{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3229fd6a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f4b7e88",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178948b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=50.09: 100%|█| 20/20 [01:26<00:00,  4.33s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWapJREFUeJzt3XlYU1f+BvA3Cwlrwh5AQXEFFFesRa2tIxWX2s1pqzJqZ2xtHaxVq7X9tTrdRqvt2Kq1Ot03O11m1FqtC3WtiqhYXBE3BBTCTsKekNzfH8jV1A0xkIS8n+fJM+bek+R7cCRvzzn3XIkgCAKIiIiInJjU1gUQERER2RoDERERETk9BiIiIiJyegxERERE5PQYiIiIiMjpMRARERGR02MgIiIiIqcnt3UBjsBsNiM3NxdeXl6QSCS2LoeIiIgaQRAElJeXIyQkBFLpzceAGIgaITc3F6GhobYug4iIiJogJycHbdu2vWkbBqJG8PLyAlD/A1WpVDauhoiIiBpDr9cjNDRU/B6/GQaiRmiYJlOpVAxEREREDqYxy124qJqIiIicHgMREREROT0GIiIiInJ6XENEREQEwGQywWg02roMuk0KheKWl9Q3BgMRERE5NUEQoNVqUVZWZutSqAmkUinCw8OhUCju6H0YiIiIyKk1hKHAwEC4u7tzA14H0rBxcl5eHsLCwu7o746BiIiInJbJZBLDkJ+fn63LoSYICAhAbm4u6urq4OLi0uT34aJqIiJyWg1rhtzd3W1cCTVVw1SZyWS6o/dhICIiIqfHaTLHZa2/OwYiIiIicnoMREREROT0GIiIiIicXPv27fH+++/b/D1siVeZ2ZAgCCitMqK4ohadNbe+Ey8REREA3HfffejVq5fVAsjBgwfh4eFhlfdyVAxENnS+qBJD/7ULHgoZjr8ez0V9RERkNYIgwGQyQS6/9Vd9QEBAC1Rk3zhlZkMhajcAQKXBBH1NnY2rISIioD5IVBnqWvwhCEKj6nvyySexa9cuLF26FBKJBBKJBBcuXMDOnTshkUiwadMm9O3bF0qlEnv27MG5c+fw0EMPQaPRwNPTE/369cOvv/5q8Z5/nO6SSCT45JNP8Mgjj8Dd3R2dO3fG+vXrb+vnmJ2djYceegienp5QqVR4/PHHkZ+fL54/cuQIhgwZAi8vL6hUKvTt2xeHDh0CAGRlZWH06NHw8fGBh4cHunXrhl9++eW2Pv92cYTIhtwUMvi4u6C0yog8XTXUbk3fUIqIiKyj2mhC1PwtLf65J9+Ih7vi1l/LS5cuxenTp9G9e3e88cYbAOpHeC5cuAAAeOmll/Duu++iQ4cO8PHxQU5ODkaOHIl//vOfUCqV+OqrrzB69GhkZGQgLCzshp/z+uuvY/HixXjnnXewfPlyJCQkICsrC76+vres0Ww2i2Fo165dqKurQ2JiIp544gns3LkTAJCQkIDevXtj5cqVkMlkSEtLEzdWTExMhMFgwO7du+Hh4YGTJ0/C09Pzlp97JxiIbCxY7VYfiMpqEBGksnU5RERk59RqNRQKBdzd3REUFHTN+TfeeAP333+/+NzX1xc9e/YUn7/55ptYu3Yt1q9fj2nTpt3wc5588kmMGzcOALBgwQIsW7YMBw4cwPDhw29Z47Zt23Ds2DFkZmYiNDQUAPDVV1+hW7duOHjwIPr164fs7GzMmTMHERERAIDOnTuLr8/OzsaYMWMQHR0NAOjQocMtP/NOMRDZWIi3K07m6ZGrq7Z1KUREBMDNRYaTb8Tb5HOtISYmxuJ5RUUFXnvtNWzcuBF5eXmoq6tDdXU1srOzb/o+PXr0EP/s4eEBlUqFgoKCRtWQnp6O0NBQMQwBQFRUFLy9vZGeno5+/fph1qxZeOqpp/D1118jLi4Ojz32GDp27AgAmD59OqZOnYqtW7ciLi4OY8aMsainOXANkY0FX15HlFdWY+NKiIgIqF8/466Qt/jDWhfW/PFqsdmzZ2Pt2rVYsGABfvvtN6SlpSE6OhoGg+Gm7/PH+4JJJBKYzWar1AgAr732Gk6cOIFRo0Zh+/btiIqKwtq1awEATz31FM6fP48JEybg2LFjiImJwfLly6322dfDQGRjwd6uAIDcMo4QERFR4ygUikbfu2vv3r148skn8cgjjyA6OhpBQUHieqPmEhkZiZycHOTk5IjHTp48ibKyMkRFRYnHunTpgpkzZ2Lr1q149NFH8fnnn4vnQkND8eyzz2LNmjV44YUX8PHHHzdrzQxENtZwpRmnzIiIqLHat2+PlJQUXLhwAUVFRTcduencuTPWrFmDtLQ0HDlyBOPHj7fqSM/1xMXFITo6GgkJCTh8+DAOHDiAiRMn4t5770VMTAyqq6sxbdo07Ny5E1lZWdi7dy8OHjyIyMhIAMCMGTOwZcsWZGZm4vDhw9ixY4d4rrkwENlYsLp+hChPxykzIiJqnNmzZ0MmkyEqKgoBAQE3XQ+0ZMkS+Pj4YMCAARg9ejTi4+PRp0+fZq1PIpHgp59+go+PDwYPHoy4uDh06NAB33//PQBAJpOhuLgYEydORJcuXfD4449jxIgReP311wHU37k+MTERkZGRGD58OLp06YIPP/ywWWuGYEO7du0SHnjgASE4OFgAIKxdu/aGbZ955hkBgPDee+9ZHC8uLhbGjx8veHl5CWq1Wvjb3/4mlJeXW7Q5cuSIMGjQIEGpVApt27YVFi1adFt16nQ6AYCg0+lu63WNkV1cKbSbu0Ho/Movgtlstvr7ExHRjVVXVwsnT54UqqurbV0KNdHN/g5v5/vbpiNElZWV6NmzJ1asWHHTdmvXrsX+/fsREhJyzbmEhAScOHECSUlJ2LBhA3bv3o0pU6aI5/V6PYYNG4Z27dohNTUV77zzDl577TV89NFHVu9PU2hUrpBIAEOdGcWVN1/gRkRERM3DppfdjxgxAiNGjLhpm0uXLuG5557Dli1bMGrUKItz6enp2Lx5Mw4ePCheZrh8+XKMHDkS7777LkJCQrB69WoYDAZ89tlnUCgU6NatG9LS0rBkyRKL4GQrCrkU/p5KFJbXIq+sBv6eSluXRERE5HTseg2R2WzGhAkTMGfOHHTr1u2a88nJyfD29rbYcyEuLg5SqRQpKSlim8GDB0OhUIht4uPjkZGRgdLS0ut+bm1tLfR6vcWjOYVcXkfEhdVERES2YdeBaNGiRZDL5Zg+ffp1z2u1WgQGBlock8vl8PX1hVarFdtoNBqLNg3PG9r80cKFC6FWq8XH1RtLNYcrexExEBER2YLQyPuIkf2x1t+d3Qai1NRULF26FF988UWL3wX+5Zdfhk6nEx9X76PQHBr2IuKVZkRELath88GqqiobV0JN1bDBpEx2Zzt92+2tO3777TcUFBRY3HjOZDLhhRdewPvvv48LFy4gKCjomm3E6+rqUFJSIt7fJSgoyOLuugDE59e7BwwAKJVKKJUtt5anjXfDXkQMRERELUkmk8Hb21v8LnF3d2/x/winpjObzSgsLIS7uzvk8juLNHYbiCZMmIC4uDiLY/Hx8ZgwYQL++te/AgBiY2NRVlaG1NRU9O3bFwCwfft2mM1m9O/fX2zzyiuvwGg0iv8lkJSUhK5du8LHx6cFe3RjnDIjIrKdhv84bux9usi+SKVShIWF3XGQtWkgqqiowNmzZ8XnmZmZSEtLg6+vL8LCwuDn52fR3sXFBUFBQejatSsAiBs2Pf3001i1ahWMRiOmTZuGsWPHipfojx8/Hq+//jomT56MuXPn4vjx41i6dCnee++9luvoLXDKjIjIdiQSCYKDgxEYGAij0Wjrcug2KRQKSKV3vgLIpoHo0KFDGDJkiPh81qxZAIBJkybhiy++aNR7rF69GtOmTcPQoUMhlUoxZswYLFu2TDyvVquxdetWJCYmom/fvvD398f8+fPt4pL7Bg2379Dqa2AyC5BJOVxLRNTSZDLZHa9DIcclEbi0/pb0ej3UajV0Oh1UKpXV399kFtD11U2oMwvY//JQBF2+DJ+IiIia7na+v+32KjNnIpNKoFFxLyIiIiJbYSCyE+JNXsu4joiIiKilMRDZieDLl97ncYSIiIioxTEQ2YmG23dc4qX3RERELY6ByE5wyoyIiMh2GIjsBKfMiIiIbIeByE407EXE23cQERG1PAYiO9GwW3VRRS2MJrONqyEiInIuDER2wtddAblUAkGoD0VERETUchiI7IRUKkGglxIAkK9nICIiImpJDER2JPDybtX5eq4jIiIiakkMRHZEo6ofISpgICIiImpRDER2RCOOEHHKjIiIqCUxENkRDafMiIiIbIKByI6Ii6rLOUJERETUkhiI7EjDCBHXEBEREbUsBiI7wikzIiIi22AgsiMNV5mVVhlRW2eycTVERETOg4HIjqjdXKCQ1/+VFPBKMyIiohbDQGRHJBLJlb2IyjltRkRE1FIYiOyMxot7EREREbU0BiI7w4XVRERELY+ByM4EqniDVyIiopbGQGRnuBcRERFRy2MgsjMNi6rzuaiaiIioxTAQ2RkuqiYiImp5DER2JpCLqomIiFocA5GdaZgyK6+pQ5WhzsbVEBEROQcGIjvjqZTDXSEDwN2qiYiIWgoDkZ2p362a02ZEREQtiYHIDgV6NVxpxhEiIiKilsBAZIe4FxEREVHLYiCyQ+JeRAxERERELYKByA4Fci8iIiKiFsVAZIcCOUJERETUohiI7JC4hoiLqomIiFoEA5Ed4mX3RERELcumgWj37t0YPXo0QkJCIJFIsG7dOvGc0WjE3LlzER0dDQ8PD4SEhGDixInIzc21eI+SkhIkJCRApVLB29sbkydPRkVFhUWbo0eP4p577oGrqytCQ0OxePHiluhekwVcvuy+ymBCZS13qyYiImpuNg1ElZWV6NmzJ1asWHHNuaqqKhw+fBjz5s3D4cOHsWbNGmRkZODBBx+0aJeQkIATJ04gKSkJGzZswO7duzFlyhTxvF6vx7Bhw9CuXTukpqbinXfewWuvvYaPPvqo2fvXVB4KGdxc6nerLqrgtBkREVFzkwiCINi6CKB+h+a1a9fi4YcfvmGbgwcP4q677kJWVhbCwsKQnp6OqKgoHDx4EDExMQCAzZs3Y+TIkbh48SJCQkKwcuVKvPLKK9BqtVAoFACAl156CevWrcOpU6caVZter4darYZOp4NKpbrjvjbGPYu3I6ekGv99NhYx7X1b5DOJiIhak9v5/naoNUQ6nQ4SiQTe3t4AgOTkZHh7e4thCADi4uIglUqRkpIithk8eLAYhgAgPj4eGRkZKC0tve7n1NbWQq/XWzxaWoBn/bQZR4iIiIian8MEopqaGsydOxfjxo0TU55Wq0VgYKBFO7lcDl9fX2i1WrGNRqOxaNPwvKHNHy1cuBBqtVp8hIaGWrs7t9SwjqiQV5oRERE1O4cIREajEY8//jgEQcDKlSub/fNefvll6HQ68ZGTk9Psn/lH/pdHiAorDC3+2URERM5GbusCbqUhDGVlZWH79u0Wc4BBQUEoKCiwaF9XV4eSkhIEBQWJbfLz8y3aNDxvaPNHSqUSSqXSmt24bRwhIiIiajl2PULUEIbOnDmDX3/9FX5+fhbnY2NjUVZWhtTUVPHY9u3bYTab0b9/f7HN7t27YTQaxTZJSUno2rUrfHx8WqYjTeDPNUREREQtxqaBqKKiAmlpaUhLSwMAZGZmIi0tDdnZ2TAajfjzn/+MQ4cOYfXq1TCZTNBqtdBqtTAY6qeRIiMjMXz4cDz99NM4cOAA9u7di2nTpmHs2LEICQkBAIwfPx4KhQKTJ0/GiRMn8P3332Pp0qWYNWuWrbrdKBwhIiIiajk2nTI7dOgQhgwZIj5vCCmTJk3Ca6+9hvXr1wMAevXqZfG6HTt24L777gMArF69GtOmTcPQoUMhlUoxZswYLFu2TGyrVquxdetWJCYmom/fvvD398f8+fMt9iqyRxwhIiIiajl2sw+RPbPFPkQ5JVW4Z/EOKOVSnHpzOCQSSYt8LhERUWvRavchciYNI0S1dWZU8PYdREREzYqByE65KWTwVNbPaHIdERERUfNiILJj/p71u2sXcS8iIiKiZsVAZMd4pRkREVHLYCCyY1cCUY2NKyEiImrdGIjs2JVL7zllRkRE1JwYiOxYwx3vOWVGRETUvBiI7Ji/FzdnJCIiagkMRHZMHCFiICIiImpWDER2TBwh4pQZERFRs2IgsmMBXlcWVfMOK0RERM2HgciO+XnUb8xoMJmhr+btO4iIiJoLA5Edc3WRQeV6+fYdFdyLiIiIqLkwENm5K5szci8iIiKi5sJAZOf8eaUZERFRs2MgsnMBvNKMiIio2TEQ2TmOEBERETU/BiI7xzveExERNT8GIjsX4MnbdxARETU3BiI7xxEiIiKi5sdAZOf8OUJERETU7BiI7Jy/V/1u1UUVBpjNvH0HERFRc2AgsnN+HvUjRCazAH2N0cbVEBERtU4MRHZOIZeKt+/gtBkREVHzYCByAP5X3fWeiIiIrI+ByAH4e3BhNRERUXNiIHIAfp71C6uLOUJERETULBiIHEDDpffFHCEiIiJqFgxEDqBhhKiQI0RERETNgoHIAXCEiIiIqHkxEDkA/4Y1RJUcISIiImoODEQOwI+37yAiImpWDEQO4MqUGUeIiIiImgMDkQNoWFRdUVuHGqPJxtUQERG1PgxEDsBLKYdCVv9XxWkzIiIi62MgcgASieTKwmpOmxEREVkdA5GD4MJqIiKi5mPTQLR7926MHj0aISEhkEgkWLduncV5QRAwf/58BAcHw83NDXFxcThz5oxFm5KSEiQkJEClUsHb2xuTJ09GRUWFRZujR4/innvugaurK0JDQ7F48eLm7prVcYSIiIio+dg0EFVWVqJnz55YsWLFdc8vXrwYy5Ytw6pVq5CSkgIPDw/Ex8ejpqZGbJOQkIATJ04gKSkJGzZswO7duzFlyhTxvF6vx7Bhw9CuXTukpqbinXfewWuvvYaPPvqo2ftnTeIIUSVHiIiIiKxNbssPHzFiBEaMGHHdc4Ig4P3338err76Khx56CADw1VdfQaPRYN26dRg7dizS09OxefNmHDx4EDExMQCA5cuXY+TIkXj33XcREhKC1atXw2Aw4LPPPoNCoUC3bt2QlpaGJUuWWASnq9XW1qK29krw0Ov1Vu757Wu40qyonCNERERE1ma3a4gyMzOh1WoRFxcnHlOr1ejfvz+Sk5MBAMnJyfD29hbDEADExcVBKpUiJSVFbDN48GAoFAqxTXx8PDIyMlBaWnrdz164cCHUarX4CA0NbY4u3paAhr2IOEJERERkdXYbiLRaLQBAo9FYHNdoNOI5rVaLwMBAi/NyuRy+vr4Wba73Hld/xh+9/PLL0Ol04iMnJ+fOO3SHxBEiLqomIiKyOptOmdkrpVIJpVJp6zIs+Hlwt2oiIqLmYrcjREFBQQCA/Px8i+P5+fniuaCgIBQUFFicr6urQ0lJiUWb673H1Z/hCPzFy+4ZiIiIiKzNbgNReHg4goKCsG3bNvGYXq9HSkoKYmNjAQCxsbEoKytDamqq2Gb79u0wm83o37+/2Gb37t0wGo1im6SkJHTt2hU+Pj4t1Js713DZfUllLUxmwcbVEBERtS42DUQVFRVIS0tDWloagPqF1GlpacjOzoZEIsGMGTPw1ltvYf369Th27BgmTpyIkJAQPPzwwwCAyMhIDB8+HE8//TQOHDiAvXv3Ytq0aRg7dixCQkIAAOPHj4dCocDkyZNx4sQJfP/991i6dClmzZplo143ja9HfSAyC0BZFUeJiIiIrMmma4gOHTqEIUOGiM8bQsqkSZPwxRdf4MUXX0RlZSWmTJmCsrIyDBo0CJs3b4arq6v4mtWrV2PatGkYOnQopFIpxowZg2XLlonn1Wo1tm7disTERPTt2xf+/v6YP3/+DS+5t1dymRQ+7i4orTKiuNIg7ktEREREd04iCALnX25Br9dDrVZDp9NBpVLZrI64JbtwtqAC3z7VHwM6+dusDiIiIkdwO9/fdruGiK7VsI6oqJJTZkRERNbEQORAxNt3lHMvIiIiImtiIHIg/pcXVnO3aiIiIutiIHIgDXsRcXNGIiIi62IgciDilBlv30FERGRVDEQORFxUzREiIiIiq2IgciB+vOM9ERFRs2AgciABlwNRYXktuH0UERGR9TAQORB/r/opsxqjGZUGk42rISIiaj0YiByIu0IOT2X93VYK9DU2roaIiKj1YCByMAFeV6bNiIiIyDoYiByMuI6Il94TERFZDQORg+EIERERkfUxEDkYBiIiIiLrYyByMAxERERE1sdA5GAaAlEBAxEREZHVMBA5GI4QERERWR8DkYPhVWZERETWx0DkYAIvjxAVV9TCZObtO4iIiKyBgcjB+HooIJEAZgEoqeRd74mIiKyBgcjByGVS+HnU39OM64iIiIisg4HIAflzHREREZFVMRA5IPHSe97glYiIyCoYiBxQoJcrAI4QERERWQsDkQPiXkRERETWxUDkgBiIiIiIrIuByAExEBEREVkXA5ED4m7VRERE1sVA5IA4QkRERGRdDEQOqCEQldfUocZosnE1REREjo+ByAGpXOVQyOv/6jhKREREdOcYiByQRCIRb/JawEBERER0xxiIHBTXEREREVkPA5GD4pVmRERE1sNA5KA4QkRERGQ9DEQO6kog4g1eiYiI7hQDkYPiCBEREZH12HUgMplMmDdvHsLDw+Hm5oaOHTvizTffhCAIYhtBEDB//nwEBwfDzc0NcXFxOHPmjMX7lJSUICEhASqVCt7e3pg8eTIqKipaujtWJa4hYiAiIiK6Y3YdiBYtWoSVK1figw8+QHp6OhYtWoTFixdj+fLlYpvFixdj2bJlWLVqFVJSUuDh4YH4+HjU1FyZSkpISMCJEyeQlJSEDRs2YPfu3ZgyZYotumQ1AbzsnoiIyGokwtXDLXbmgQcegEajwaeffioeGzNmDNzc3PDNN99AEASEhITghRdewOzZswEAOp0OGo0GX3zxBcaOHYv09HRERUXh4MGDiImJAQBs3rwZI0eOxMWLFxESEnLN59bW1qK29krQ0Ov1CA0NhU6ng0qlauZeN87F0ioMWrQDCpkUGW8Nh0QisXVJREREdkWv10OtVjfq+9uuR4gGDBiAbdu24fTp0wCAI0eOYM+ePRgxYgQAIDMzE1qtFnFxceJr1Go1+vfvj+TkZABAcnIyvL29xTAEAHFxcZBKpUhJSbnu5y5cuBBqtVp8hIaGNlcXm8zPo36EyGAyo7y2zsbVEBEROTa5rQu4mZdeegl6vR4RERGQyWQwmUz45z//iYSEBACAVqsFAGg0GovXaTQa8ZxWq0VgYKDFeblcDl9fX7HNH7388suYNWuW+LxhhMieuClk8FDIUGkwobjCAJWri61LIiIiclh2HYh++OEHrF69Gt9++y26deuGtLQ0zJgxAyEhIZg0aVKzfa5SqYRSqWy297cWP08lKkuqUFRRi3B/D1uXQ0RE5LDsOhDNmTMHL730EsaOHQsAiI6ORlZWFhYuXIhJkyYhKCgIAJCfn4/g4GDxdfn5+ejVqxcAICgoCAUFBRbvW1dXh5KSEvH1jsrPU4HskioUc7dqIiKiO9KkNURffvklNm7cKD5/8cUX4e3tjQEDBiArK8tqxVVVVUEqtSxRJpPBbDYDAMLDwxEUFIRt27aJ5/V6PVJSUhAbGwsAiI2NRVlZGVJTU8U227dvh9lsRv/+/a1Wqy34X770vqjCYONKiIiIHFuTAtGCBQvg5uYGoH7R8ooVK7B48WL4+/tj5syZVitu9OjR+Oc//4mNGzfiwoULWLt2LZYsWYJHHnkEQP1d32fMmIG33noL69evx7FjxzBx4kSEhITg4YcfBgBERkZi+PDhePrpp3HgwAHs3bsX06ZNw9ixY697hZkj8fdUAACKGYiIiIjuSJOmzHJyctCpUycAwLp16zBmzBhMmTIFAwcOxH333We14pYvX4558+bh73//OwoKChASEoJnnnkG8+fPF9u8+OKLqKysxJQpU1BWVoZBgwZh8+bNcHV1FdusXr0a06ZNw9ChQyGVSjFmzBgsW7bManXaSsOVZsWVnDIjIiK6E03ahygwMBBbtmxB79690bt3b8yaNQsTJkzAuXPn0LNnT4ffBfqPbmcfg5b0+d5MvP7zSYyMDsKHCX1tXQ4REZFduZ3v7yaNEN1///146qmn0Lt3b5w+fRojR44EAJw4cQLt27dvyltSE3ANERERkXU0aQ3RihUrEBsbi8LCQvzvf/+Dn58fACA1NRXjxo2zaoF0Y37iGiJOmREREd2JJo0QeXt744MPPrjm+Ouvv37HBVHjNYwQFVdyhIiIiOhONGmEaPPmzdizZ4/4fMWKFejVqxfGjx+P0tJSqxVHN+fnUT9CVFZlhNFktnE1REREjqtJgWjOnDnQ6/UAgGPHjuGFF17AyJEjkZmZaXHLC2pe3u4KSC/f07WEo0RERERN1qQps8zMTERFRQEA/ve//+GBBx7AggULcPjwYXGBNTU/mVQCXw8liipqUVRRC43K9dYvIiIioms0aYRIoVCgqqoKAPDrr79i2LBhAABfX19x5IhaBjdnJCIiunNNGiEaNGgQZs2ahYEDB+LAgQP4/vvvAQCnT59G27ZtrVog3Zx4pRk3ZyQiImqyJo0QffDBB5DL5fjvf/+LlStXok2bNgCATZs2Yfjw4VYtkG5O3K2aI0RERERN1qQRorCwMGzYsOGa4++9994dF0S3p+HS+0LuRURERNRkTQpEAGAymbBu3Tqkp6cDALp164YHH3wQMpnMasXRrflxDREREdEda1IgOnv2LEaOHIlLly6ha9euAICFCxciNDQUGzduRMeOHa1aJN2YP3erJiIiumNNWkM0ffp0dOzYETk5OTh8+DAOHz6M7OxshIeHY/r06daukW7iyh3vOUJERETUVE0aIdq1axf2798PX19f8Zifnx/efvttDBw40GrF0a01TJkVlXOEiIiIqKmaNEKkVCpRXl5+zfGKigooFIo7LooaT7zjfaUBgiDYuBoiIiLH1KRA9MADD2DKlClISUmBIAgQBAH79+/Hs88+iwcffNDaNdJNNIwQGerMqKits3E1REREjqlJgWjZsmXo2LEjYmNj4erqCldXVwwYMACdOnXC+++/b+US6WbcFXK4K+qv7OOVZkRERE3TpDVE3t7e+Omnn3D27FnxsvvIyEh06tTJqsVR4/h5KlBVUo3iylq09/ewdTlEREQOp9GB6FZ3sd+xY4f45yVLljS9Irpt/p5K5JRUo7CcI0RERERN0ehA9PvvvzeqnUQiaXIx1DRXLr3nlWZERERN0ehAdPUIENkX3vGeiIjozjRpUTXZFz/uVk1ERHRHGIhagYYpsyLuVk1ERNQkDEStgL/X5UDE3aqJiIiahIGoFfD3uDxlxhEiIiKiJmEgagUaRogK9DU2roSIiMgxMRC1AsFqVwCAvqYOlbx9BxER0W1jIGoFvFxd4OVav4NCnq7axtUQERE5HgaiViJE7QYAuFTGaTMiIqLbxUDUSoR410+b5ZVxhIiIiOh2MRC1EiHe9SNEuQxEREREt42BqJVoCEScMiMiIrp9DESthDhlxkXVREREt42BqJVoWFTNKTMiIqLbx0DUSohriHQ1EATBxtUQERE5FgaiVkKjcoVEAhjqzLyFBxER0W2y+0B06dIl/OUvf4Gfnx/c3NwQHR2NQ4cOiecFQcD8+fMRHBwMNzc3xMXF4cyZMxbvUVJSgoSEBKhUKnh7e2Py5MmoqKho6a40K4VcigDP+lt4cNqMiIjo9th1ICotLcXAgQPh4uKCTZs24eTJk/jXv/4FHx8fsc3ixYuxbNkyrFq1CikpKfDw8EB8fDxqaq5cbZWQkIATJ04gKSkJGzZswO7duzFlyhRbdKlZXbn0nleaERER3Q65rQu4mUWLFiE0NBSff/65eCw8PFz8syAIeP/99/Hqq6/ioYceAgB89dVX0Gg0WLduHcaOHYv09HRs3rwZBw8eRExMDABg+fLlGDlyJN59912EhIS0bKeaURtvN6TllHGEiIiI6DbZ9QjR+vXrERMTg8ceewyBgYHo3bs3Pv74Y/F8ZmYmtFot4uLixGNqtRr9+/dHcnIyACA5ORne3t5iGAKAuLg4SKVSpKSkXPdza2trodfrLR6OoOEmrwxEREREt8euA9H58+excuVKdO7cGVu2bMHUqVMxffp0fPnllwAArVYLANBoNBav02g04jmtVovAwECL83K5HL6+vmKbP1q4cCHUarX4CA0NtXbXmkXDlFmejlNmREREt8OuA5HZbEafPn2wYMEC9O7dG1OmTMHTTz+NVatWNevnvvzyy9DpdOIjJyenWT/PWq7sVs0RIiIiotth14EoODgYUVFRFsciIyORnZ0NAAgKCgIA5OfnW7TJz88XzwUFBaGgoMDifF1dHUpKSsQ2f6RUKqFSqSwejqBht2pOmREREd0euw5EAwcOREZGhsWx06dPo127dgDqF1gHBQVh27Zt4nm9Xo+UlBTExsYCAGJjY1FWVobU1FSxzfbt22E2m9G/f/8W6EXLaRghKqyohaHObONqiIiIHIddB6KZM2di//79WLBgAc6ePYtvv/0WH330ERITEwEAEokEM2bMwFtvvYX169fj2LFjmDhxIkJCQvDwww8DqB9RGj58OJ5++mkcOHAAe/fuxbRp0zB27NhWdYUZAPh5KKCQSyEIQL6e64iIiIgay64DUb9+/bB27Vr85z//Qffu3fHmm2/i/fffR0JCgtjmxRdfxHPPPYcpU6agX79+qKiowObNm+Hq6iq2Wb16NSIiIjB06FCMHDkSgwYNwkcffWSLLjUriUSCEF5pRkREdNskAm98dUt6vR5qtRo6nc7u1xON/3g/9p0rxntP9MQjvdvauhwiIiKbuZ3vb7seIaLbx92qiYiIbh8DUSvDKTMiIqLbx0DUylwZIWIgIiIiaiwGolaGU2ZERES3j4GolWnv5wEAyCyqRG2dycbVEBEROQYGolYm1NcNfh4KGExmHL/kGDelJSIisjUGolZGIpGgTzsfAMDhrFIbV0NEROQYGIhaob6XA1EqAxEREVGjMBC1QmIgyi4F990kIiK6NQaiVii6jRouMgkKy2txsZSX3xMREd0KA1Er5OoiQ7cQNQBOmxERETUGA1ErxXVEREREjcdA1EoxEBERETUeA1Er1RCITmn1qKits3E1RERE9o2BqJXSqFzRxtsNZgE4klNm63KIiIjsGgNRK8ZpMyIiosZhIGrFGIiIiIgah4GoFWsIRIezS2E2c4NGIiKiG2EgasUigrzg5iJDeU0dzhZW2LocIiIiu8VA1IrJZVL0CvUGwGkzIiKim2EgauW4joiIiOjWGIhaOXEdEQMRERHRDTEQtXK9w7wBAOeLKlFSabBtMURERHaKgaiV83ZXoFOgJwCOEhEREd0IA5ET6Bt2eR1RNgMRERHR9TAQOQEurCYiIro5BiIn0OdyIDqSUwajyWzjaoiIiOwPA5ET6ODvAW93F9TWmXEyV2/rcoiIiOwOA5ETkEol6BPGaTMiIqIbYSByEuI6Ii6sJiIiugYDkZNo2I8oLbvMpnUQERHZIwYiJ9EtWA0AuFRWDX2N0cbVEBER2RcGIiehdndBiNoVAJChLbdxNURERPaFgciJRASrAACn8nilGRER0dUYiJxI1yAvAEA6R4iIiIgsMBA5kYjLgYhTZkRERJYYiJxI5OUpswxtOcxmwcbVEBER2Q+HCkRvv/02JBIJZsyYIR6rqalBYmIi/Pz84OnpiTFjxiA/P9/iddnZ2Rg1ahTc3d0RGBiIOXPmoK6uroWrt71wfw8oZFJU1NbhUlm1rcshIiKyGw4TiA4ePIh///vf6NGjh8XxmTNn4ueff8aPP/6IXbt2ITc3F48++qh43mQyYdSoUTAYDNi3bx++/PJLfPHFF5g/f35Ld8HmXGRSdAr0BACkc2E1ERGRyCECUUVFBRISEvDxxx/Dx8dHPK7T6fDpp59iyZIl+NOf/oS+ffvi888/x759+7B//34AwNatW3Hy5El888036NWrF0aMGIE333wTK1asgMFgsFWXbKZhHdEpriMiIiISOUQgSkxMxKhRoxAXF2dxPDU1FUaj0eJ4REQEwsLCkJycDABITk5GdHQ0NBqN2CY+Ph56vR4nTpy47ufV1tZCr9dbPFqLiGAurCYiIvojua0LuJXvvvsOhw8fxsGDB685p9VqoVAo4O3tbXFco9FAq9WKba4OQw3nG85dz8KFC/H6669boXr7ExFUv7A6Xdt6Qh4REdGdsusRopycHDz//PNYvXo1XF1dW+xzX375Zeh0OvGRk5PTYp/d3BpGiC4UVaLaYLJxNURERPbBrgNRamoqCgoK0KdPH8jlcsjlcuzatQvLli2DXC6HRqOBwWBAWVmZxevy8/MRFBQEAAgKCrrmqrOG5w1t/kipVEKlUlk8WosATyX8PBQwC8CZAk6bERERAXYeiIYOHYpjx44hLS1NfMTExCAhIUH8s4uLC7Zt2ya+JiMjA9nZ2YiNjQUAxMbG4tixYygoKBDbJCUlQaVSISoqqsX7ZGsSiUTcsfpUHgMRERERYOdriLy8vNC9e3eLYx4eHvDz8xOPT548GbNmzYKvry9UKhWee+45xMbG4u677wYADBs2DFFRUZgwYQIWL14MrVaLV199FYmJiVAqlS3eJ3sQEaTCvnPFvNKMiIjoMrsORI3x3nvvQSqVYsyYMaitrUV8fDw+/PBD8bxMJsOGDRswdepUxMbGwsPDA5MmTcIbb7xhw6ptq2Ed0daTWoT5uuHujn7oEugFqVRi48qIiIhsQyIIAu/hcAt6vR5qtRo6na5VrCc6V1iBuCW7cPXfvJerHNFt1OjR1hv92vvgrnBfeLm62K5IIiKiO3Q7398MRI3Q2gIRAJzS6rEtvQD7zxfj0IVSVBstrziTSSXo0VaN5/7UCX+K0NzgXYiIiOwXA5GVtcZAdDWjyYzT+eU4elGHtOwy7M8sRlZxFQCgg78Hts++z7YFEhERNcHtfH87/BoiunMuMim6hajRLUSNcXeFAQBO55dj2Hu7kVlciYraOngq+X8VIiJqvez6snuynS4aL2hUSggCkMFdrYmIqJVjIKIbigquH148yf2KiIiolWMgohuKCrkciHI5QkRERK0bAxHdUFSwGgBwMo+BiIiIWjcGIrqhhhGiU3l61JnMNq6GiIio+TAQ0Q2183WHu0KG2jozLhRX2rocIiKiZsNARDcklUoQcflGsCe4joiIiFoxBiK6KXFh9VXriIoqamE2cz9PIiJqPRiI6KbEhdWXR4iSzxVjwNvb8cDyPSiqqLVlaURERFbDQEQ31TBClJ5XDrNZwFsbT8JQZ8bJPD2e+Hcy8vU1Nq6QiIjozjEQ0U111XhBKqmfJvt0TyZO5OrhpZQjWO2Kc4WVePzfybhYWmXrMomIiO4IAxHdlJtChg4BngCAxVtOAQCeva8jfngmFqG+bsgqrsLfVx/mmiIiInJoDER0Sw238DCaBGhUSvxtYDhCfd3x/ZRYeChkOHpRh43H8mxcJRERUdMxENEtRV4ORAAwM64L3BQyAECItxumDO4IAHh3awaMf9i80WQWcLagHFtPaFFaaWi5gomIiG6T3NYFkP27K9wXQP16oj/3bWtx7ql7wvH1/gvIKq7CdweyMSG2PX7PLsWizadwJEeHaqNJfI/vp9wNiUTS4vUTERHdCgMR3VLfdj744ZlYdAzwgFxmOajooZTj+aGdMe+nE1i67QxySqvxyW/n0bCkyM1FBqPJjAOZJdh9pgj3dgmwQQ+IiIhujlNm1Ch3hfvCz1N53XNj7wpDez93FFUY8NHu+jD0SO82+HXWYBx/PR4TY9sDAJZszYAgcPE1ERHZHwYiumMuMileGhEBAPD3VOKjCX3x3hO90CnQCzKpBFPv6wg3FxmOXNRhW3qBjaslIiK6FgMRWcXw7sFImjkYO2bfi2HdgizOBXgpMXFAOwDAkqTTvESfiIjsDgMRWU1njRe8XF2ue+6ZwR3hoZDhZJ4e36RkwcRQREREdoSBiFqEr4cCfxsUDgCY/9MJDHx7O97Zcgq6aqONKyMiImIgohY07U+dMGVwB/i4u0Crr8GKHecw6/s0W5dFRETEQEQtRymX4f9GRmL//w3F0rG9IJEA204V4EJRpa1LIyIiJ8dARC1OKZfhoV5tcN/lPYm+2Z9l44qIiMjZMRCRzTTsT/TDoRxUG0y2LYaIiJwaAxHZzOAuAQj1dYO+pg4/H8m1dTlEROTEGIjIZmRSCf7Sv35/oq/2X+Au1kREZDMMRGRTj8eEQimX4vglPQ5eKLV1OURE5KR4c1eyKR8PBUb3DMF/Uy/i8X8nI1jtim4hKjx9Twf07+Bn6/KIiMhJcISIbG7qfR3ROdATAJCnq8Gv6QVI+CQFPxzKsXFlRETkLCQCF27ckl6vh1qthk6ng0qlsnU5rVZ5jRGntOX4ct8FbDiaBwD4+30d0TXIC/vOFuN4rg5TBnfAQ73a2LhSIiJyBLfz/c0pM7IbXq4u6NfeFzHtfBDu74Hl28/iw53nLNrM/vEIglSuFtNpumoj1G7Xv4caERFRY3DKjOyORCLBC8O64p0/94CXq/zymqJwxEUGwmgSMHX1YeSUVKFAX4Op36Si5+tbsSTptK3LJiIiB8Yps0bglJl9qDaY8Ni/9+H4JT3a+bmjpNKA8po6AICLTIKtM+9FuL+HjaskIiJ7cTvf33Y9QrRw4UL069cPXl5eCAwMxMMPP4yMjAyLNjU1NUhMTISfnx88PT0xZswY5OfnW7TJzs7GqFGj4O7ujsDAQMyZMwd1dXUt2RWyAjeFDB9PjEGAlxJZxVUor6lDj7Zq9GvvA6NJwIJf0m1dIhEROSi7DkS7du1CYmIi9u/fj6SkJBiNRgwbNgyVlVduBjpz5kz8/PPP+PHHH7Fr1y7k5ubi0UcfFc+bTCaMGjUKBoMB+/btw5dffokvvvgC8+fPt0WX6A4Fq93w2aR+GNDRD6+OisSaqQOw8NFoyKQSJJ3Mx75zRbYukYiIHJBDTZkVFhYiMDAQu3btwuDBg6HT6RAQEIBvv/0Wf/7znwEAp06dQmRkJJKTk3H33Xdj06ZNeOCBB5CbmwuNRgMAWLVqFebOnYvCwkIoFIpbfi6nzOzf/J+O46vkLEQGq7DhuUGoNppQVVuHAC8lJBKJrcsjIiIbaDVTZn+k0+kAAL6+vgCA1NRUGI1GxMXFiW0iIiIQFhaG5ORkAEBycjKio6PFMAQA8fHx0Ov1OHHixHU/p7a2Fnq93uJB9m1GXBd4ucqRnqdH5LzN6P6PLbhrwTYM/dcuLNt2BjklVbYukYiI7JjDBCKz2YwZM2Zg4MCB6N69OwBAq9VCoVDA29vboq1Go4FWqxXbXB2GGs43nLuehQsXQq1Wi4/Q0FAr94aszddDgdnDugIADCazePx8USWWJJ3GPYt3YM6PR6CrMt7wPUxmhxksJSIiK3OYfYgSExNx/Phx7Nmzp9k/6+WXX8asWbPE53q9nqHIAUyMbYdBnf2hkEnh56mAIACbj2ux9vdL2HuuCD+mXsTO04V46+HuiO8WZPHa7w5k4/WfT2JibDvMHR4BqZTTbEREzsQhRoimTZuGDRs2YMeOHWjbtq14PCgoCAaDAWVlZRbt8/PzERQUJLb541VnDc8b2vyRUqmESqWyeJD9k0gk6BjgiVBfd7gr5PBQyjGmb1t881R//PhMLDoEeKCwvBbPfJ2KFTvOiq+7WFqF138+iWqjCf/efR5TV6ei2mCyYU+IiKil2XUgEgQB06ZNw9q1a7F9+3aEh4dbnO/bty9cXFywbds28VhGRgays7MRGxsLAIiNjcWxY8dQUFAgtklKSoJKpUJUVFTLdIRsLqa9L36Zfg+eGdwBAPDu1gzsOFUAQRAw/6cTqDaa0CHAAwqZFFtO5OOJj5JRVFFr46qJiKil2PVVZn//+9/x7bff4qeffkLXrl3F42q1Gm5ubgCAqVOn4pdffsEXX3wBlUqF5557DgCwb98+APWX3ffq1QshISFYvHgxtFotJkyYgKeeegoLFixoVB28yqx1eXXdMXyzPxsqVzn+PqQT3t50Ci4yCTY9PxilVQZM+eoQSquM6BXqje+m3A1XF5mtSyYioia4ne9vuw5EN7pc+vPPP8eTTz4JoH5jxhdeeAH/+c9/UFtbi/j4eHz44YcW02FZWVmYOnUqdu7cCQ8PD0yaNAlvv/025PLGLaFiIGpdDHVmPPFRMn7PLhOPTR/aGbPu7wIAOFdYgUc/3AddtREP9wrBe0/04qX7REQOqNUEInvBQNT6aHU1eGD5byiqMCDc3wObnr/HYiRo39kiTPjsAExmAXPiu+LBniE4pS1HYXktHuoVAg+lw1yPQETktBiIrIyBqHVKyynD0l9PY+b9XdCjrfc157/en4V5645fc3xI1wB8MqkfZLwSjYjIrrXajRmJrKlXqDc+/+td1w1DADDh7nb468D2AOpvHhsR5AWlXIodGYV4d2vGdV9DRESOiSNEjcARIuclCALydDUI8FLCRSbFT2mX8Px3aQCAZeN648GeIbYtkIiIbogjRERWIpFIEOLtBhdZ/T+Vh3q1wTP31l+6/+J/j+Bwdmmj3uf4JR2+3p8FXfWNd8omIiLb4QhRI3CEiK5mMguY/OVB7MwohIdChk8m9UNsR79r2tWZzFh/JBdfJWchLacMAODvqcAroyLxcK82vHKNiKiZcVG1lTEQ0R9V1tZhyteHsPdsMRRyKVYm9MHQyCv3zBMEAdO/S8PPR3IB1K9BCvBUIldXAwC4q70vxvUPxZCugfB2V9ikD0RErR0DkZUxENH11BhNeO4/vyPpZD7kUgmWju2NUT2CAQBrDl/ErB+OQCaV4PmhnTHurjCo3VzwyZ7zWLbtDGqM9TeglUklGNDRD/96rCcCVa627A4RUavDQGRlDER0I0aTGXN+PIJ1abmQSSVYPq43erRVY8T7v6G8tg6z7u+C6UM7W7zmYmkVvjuQg1/T83FKWw4AGBkdhA8T+tqiC0RErRYDkZUxENHNmMwC5vx4BGt+vwSZVIJwfw+cLahAnzBv/PBMLOSyG1+78Ht2Kcas3AezAHw35W7c3eHatUhERNQ0vMqMqAXJpBK881hPPNq7DUxmAWcLKuChkOG9J3rdNAwBQO8wH4y7KwwA8MbPJ2EyX/nvE5NZQGVtHYoqaqGrsrw6rdpgwie/nUfi6sNYuCkdPxzKQXqe3vqdIyJyErz/AJEVNIQiuUyCdb/n4q1HuqOdn0ejXjvr/i5YfyQXJ/P0+PFQDjprPLFs21nsPlOIq8dvu7dR4U8RGrgrZPjkt/MoqjBc816jegRj/gNR0HA9EhHRbeGUWSNwyoxuR43RZHFftMb45LfzeGtjOlxkEhhNjfsnGerrhsf7hqKwohZn8iuQklkMswB4KuWYPawLEu5uJ+6fRETkjLiGyMoYiKi5GerMGP7+bpwvqoRcKsGf+7bFU/d0QBtvNyjlUhRXGrAzowDb0gtQWFGLx2Pa4tE+bS0Cz/FLOryy7jiOXN7zKNzfAy8M64KR3YNRVm3EsUs6HLtYdvl/dTCYBIztF4onB7aHv6fSRj0nImo+DERWxkBELeFCUSW2nNBiVI9gtPVxb9J7mMwCvj2QjfeSTqOksn5KTe3mctMdspVyKR6PCcXsYV2hdne55WfUmcyoMwu3PQpGRNTSGIisjIGIHE1FbR0++e08Pt59HpUGE4D6EaPoNur6R1s1SisNWLX7vDiiFOilxIJHohEXpRHv4SaXSRDodWU9UmF5LSZ9dgDnCiswZXAHPHtvR3go65ci1hhNqDKY4OPuAolEgjqTGdtOFeCHgzmoMwt4ZnAHDOjk3+I/CyJyXgxEVsZARI6qrMqAc4WV6KzxhMr12tEfQRCQfL4Y89Ydx7nCSgBAz7ZqXCytRnGlARIJMOWeDph5fxfoq40Y9/F+sR0ABHgpEd9Ng2OX9DhxSYc6swBvdxd0CvDExdJqaPU1Fp93X9cAzB0egchg/jsioubHQGRlDETU2tUYTXgv6TQ+/u08Gq78l0kl4jYAXTVeMJrMOF9UiWC1K6YP7YxVu84hq7jqpu/r66HA4zGhqDbUYXVKNurMAiQS4NHebfHCsC4IUrliz9kirP39ElxkEjz3p84I9b3+dKHRZMbRi2XYd7YYKZklULnJMXlQOPq287Xqz4KIWg8GIitjICJncTJXj+OXdOgS5IWIIC/sPl2I/1t7TLzEv423G/7z9N0I83OHoc6M7w9m41xhJXqFeqNvOx/4eypxvqgCZwsqoJTLMCQiAEp5/VqjC0WVeGdLBjYeywMAKORS+HsoxPu7AYCrixTTh3bGU4M6QCGvXzBuNJnx46GLWLrtNPL1tdfUfHcHX0wb0hkDO/k1+w1zddVGFFfUQiaVQCqRQKNyFeskIvvDQGRlDETkzIoravHWxnRcLK3Cksd73XAEp7HScsqw8Jd0pGSWAABUrnI83LsNTueXY//5+mP+ngp0DfJCB39P7DlbhMyi+mk6b3cXxHbwQ2xHP5zM1eN/hy+K2xT0DPXGtCGdEBcZ2Ohg1LBW6nR+Oc4VViJE7YqhkZprQk6N0YSVO89h5c5zMJjM4nFPpRz3dg3AsCgN7o/SwF3Brd2I7AkDkZUxEBFZlyAI2H++BLpqI+7rGgBXFxkEQcCaw5fwz1/SxSvkGvh5KDDtT50wvn+YOOIEALll1fho93n850A2auvqg4q/pxIdAzzQIcADgASXyqqRV1aNzhpPvDIqCm283QAA+84W4aU1x5BdUnXNZ/05pi1iLk/FldcYsWzbGVy4PD3oqZRDEAQYzQIMdVfCUaivGz6b1A+dNV5W/3kRUdMwEFkZAxFRy6k2mJCu1eNsQQXOFVYgwFOJsXeFwVN549GXoopafLonE18nZ6Gitu6G7TwUMrw0IgIXiqvw6Z5MAID88v3nwv09cORi2XWn5QBAo1LiH6O7YUT3IEgkEpjNAo5cLMPWk/lY9/sl5Olq4KWU44OEPogI8sJXyRfw39SLuCvcD/96rGeLTa2VVBqwPu0ShkQENnq3dKLWioHIyhiIiBxDlaEOZ/IrkFlUifOXp9naervB10OBlbvOITWr1KL9+P5heGVkpLh1QJ3JjO2nCvDf1IsorLgSjPq198Vzf+oEr+tcqQfUh5Bnv07FgQslkEoAqUSCuqvuSxffTYMPxve54c7hpZUGpGaV4uglHbzdXNC9jRpdNV44XVCO3acLcTi7FJ0DvTC+fxi6XB6BKqk04MjFMqhcXdAp0BNKuRSf7c3Eyh3nUF5bB39PJf43NbZZQ1FtnQl7zhShX7jvda9iJLI1BiIrYyAicnwms4Cvki9g8eYMeCjlWPznaPwpQmO196+tM+GVtcfx39SLAIC72vviT5GBWLL1NAwmMx7oEYxpf+qEPWeKsO9cMYorDagxmFBRW4dLZdWN/pw+Yd6oMphwSltucVwhl4pTeA1/DvN1x3+nxlrsJSUIAnZmFOKb/VkY0MkfCf3DxE02a+tMOJGrR0d/z1tu0mk2C3j2m1RsPZmPNt5uWD6+N/qE+dywfY3RhE/3ZGLLCS1mxnXBkIhA8Zy+xoi9Z4pwT5eAm44EEt0uBiIrYyAiaj3Ka4xwkUmbZadtQRCw83QhAjyV6N5GDQDYfiofz3ydest71HUM8EDvMB+UVRlxIleHPF0NvN1dMKiTP/q198W+c0X4Nb1A3AoBADoEeKCq1iTu99TG2w2z47tgYEd//HlVMrJLqhAZrMLbj0ZD5eYCfbUR727NwG9nisT3CFa74ql7OuBcYQU2HMmFvqYOLjIJBncOQFyUBkXltTh2qb6eR/u0wZMD2kMikWDpr2fw3q+nxfeRSyWYHd8VU+7pAKn0yqJ2s1nAz0dzsXhzhhj8FHIpvniyHwZ08selsmpM/DQF5wor0dbHDYv/3AMDOt56A88qQ51VF7FX1NahQF+DDgGeVntPsj0GIitjICKiO7HlhBbPffs7JBKgfwc/DO7sj/Z+HnBTyODqIkW4vyd8PRQWr9HXGOGhkEN2VbjQ6mqQdFILf08l7gr3hd/le9BV1NYht6wa7fzcxUXnWcWVGLMyGUUV166JUsikeLh3CH47U4Q8neXmmZ5K+U3XYQ3pGoAR0cF48b9HAQD/GB2F1KxSbDhav51CqK8bxt/VDvdHabAtPR/fHsgW96sKUbsizM8d+8+XwF0hw5sPdcc7WzKu2cAzoX8YhnQNRJDaFW283eBz1c8mu7gK/7f2GPadK8Kz93bEC8O6QiaVoMZowj83puPno7mIi9RgfP8w9A71vukVh4Ig4MhFHb47kI31R3JRZTDhmcEdMHd4hEWou5UaownFlQZxwT7ZDwYiK2MgIqI7VVZlgKuLrEXvAXcyV49/rD+Oi6XVqKipQ22dGXFRgZg7PALt/DxQYzRhdUo21qddQqdALzzapw3u7uCH84UV2HA0D8nnihHs7YroNmrUmQUsSTptcWXdkwPa47UHu0EQBHx3MAcLf0mHvubaMOXlKseUezrg6cEdIJEAk784hD1nr4xSdQr0xKq/9MHney9gdUr2Na+PClZhaGQgXF1kWL79DGqMV2r4U0QgXhjWBXP/dxTHL+ktXhcR5IWR0cG4P0qDiCAvMRzpqo34Ke0S/nMgB+l5lq8BgBHdg/DeE70AAIezSvF7ThlO55fjdH4F6kxmTB4UjsdiQiGTSrAjowCvrDmGXF0NRvUIxovxXS3WbQmCgOySKhy5qEO+rgauChncXGTwca9f+xXq437d8GU2C6gzC3CRSZp9f60bOXZRBxe5BBFBTf/eqzGa8N2BbAR7uyG+W5AVq2scBiIrYyAiotZAEIQ7+nJNz9Pjuf/8jrMFFbi7gy++ntzfYqF4tcGEn4/mYvX+LBy5qEOPtmok9A/D6J4hFtNblbV1mPBpCg5nl6FPmDc+e7IfvN3rR4H2ni3C18lZ9dsl6GquO8IV28EPw7pp8PamU+J2CwDg4+6CF4dH4OCFEmw8mmdxzt9TIYbRwvJa8ZxCLsXI7kEYd1cYLpVVY+7/jsJoEhCkckVJpcFi36mrRQR5oVOgpzgy1kAuleD+KA0MdWYUVRqQVVyJsqob31zZ1UWKNt5ucHWRQSmXos4soLC8FoXltagzC5BKADcXGdr7e2D2sK4Wa68aCIKAkkoDPJTyawL33rNFOJBZgipDHSoNJrhIJQhUuSLQS4lQX3dEBHmJP/sGF0ursPCXU9h4LA9SCTA7viueHdzxtkbNACDlfDFeXnsM5y/f7uepQeF4eWSkxahnc2MgsjIGIiKietUGE/adK8KAjv5wU9x4tKvGaLrpaFi1wYSUzGLc3cHvpu2KK2qxM6MQ207l42JpNRL6h+HxmFBIJBIcu6jDlK8PIU9Xg16h3vgwoQ9CLk9blVUZsOWEFkkn8/HbmSKLcATU345m7F2heKR3G4tAsP98MZ75OhW66voQo1EpcVe4HyKDvdAl0AsXiiuxfPtZ8bxUAvxtYDhG9QjG0m1nsDOj8Jo+KGRSRIao0N7PHbVGM6qMJhToa3C+qNJixK0x7u0SgMdi2uK0thxHL+lwvrASWl0NDCYzPJVyvDIqEmP7hcJkFrB4SwY+2n3+lu8ZrHZFqI87PF3lUMql2H6q4JqfV1xkIJ66pwN+zy7D4exSqN1cMO6uMPQJs5yW1FUZse9cETaf0OKntFwA9RuqNoTCIV0DsHRcb4urEhu2iqgzC3jqng639fO4FQYiK2MgIiKyT6WVBhzOLsU9nQNuuNdTw3YMDV92nkoZOgZ43nC0LKekCgcvlKB3mA/a+7lf066syoAPd55Dep4es+7vgt5XXV23/3wxDmeXwsddAT8PBUK83dBZ42mxoWiDOpMZ2SVV0OprYKgzw1BnhkwqQYCXEgFeSrgr5Kg11l+J+P3BHHy2N/OWi/OB+tBkNJmx71wxAOCBHsFo4+0GN4UMhjozCsprka+vQWZRJS6WXv8Kx/7hvnjtwW5IyynDP9afuGFw6xaiQnQbNbT6GuSV1eBMQTmuWveP8f3DMHd4BH47U4gXfjiC2joz3BUy9A7zRt92vsjQ6rH9VAGMJgE+7i5I+b84q+7ZxUBkZQxERERka5lFlXh3SwYyiyoRFaJCz7ZqdA1SIcTbFf6eSnyzPwuLt2SI4cVdIcO7j/XEyOjgG76nvsaI09py5OtrUVFrRHlNHToEeGBI1yu3wDl2UYcX/3cUBfoa9G3ng37tfXE6vxzrj+ReM5IE1K8Ju6ezPx7sGWIRFo9eLEPit4eRU3JtCItuo8aYPm0w9q4wq66zYyCyMgYiIiJyBGcLyvHquuOorDVhyeM9m/VWMqWVBvyUdgll1UYEq12hUbmia5AXgtU3vtrObBZwpqACBy6U4PfsUgR4KvFon7boGtQ8dTIQWRkDERERkeO5ne/vlrm5DhEREZEdYyAiIiIip8dARERERE6PgYiIiIicHgMREREROT2nCkQrVqxA+/bt4erqiv79++PAgQO2LomIiIjsgNMEou+//x6zZs3CP/7xDxw+fBg9e/ZEfHw8CgoKbF0aERER2ZjTBKIlS5bg6aefxl//+ldERUVh1apVcHd3x2effWbr0oiIiMjGnCIQGQwGpKamIi4uTjwmlUoRFxeH5OTka9rX1tZCr9dbPIiIiKj1copAVFRUBJPJBI1GY3Fco9FAq9Ve037hwoVQq9XiIzQ0tKVKJSIiIhtwikB0u15++WXodDrxkZOTY+uSiIiIqBnJbV1AS/D394dMJkN+fr7F8fz8fAQFBV3TXqlUQqlUtlR5REREZGNOMUKkUCjQt29fbNu2TTxmNpuxbds2xMbG2rAyIiIisgdOMUIEALNmzcKkSZMQExODu+66C++//z4qKyvx17/+1dalERERkY05TSB64oknUFhYiPnz50Or1aJXr17YvHnzNQutr0cQBADg1WZEREQOpOF7u+F7/GYkQmNaObmLFy/ySjMiIiIHlZOTg7Zt2960DQNRI5jNZuTm5sLLywsSicTW5VidXq9HaGgocnJyoFKpbF1Os3Om/jpTXwH2tzVzpr4C7K+1CIKA8vJyhISEQCq9+bJpp5kyuxNSqfSWybI1UKlUTvEPr4Ez9deZ+gqwv62ZM/UVYH+tQa1WN6qdU1xlRkRERHQzDERERETk9BiICEqlEv/4xz+cZjNKZ+qvM/UVYH9bM2fqK8D+2gIXVRMREZHT4wgREREROT0GIiIiInJ6DERERETk9BiIiIiIyOkxEDmJhQsXol+/fvDy8kJgYCAefvhhZGRkWLSpqalBYmIi/Pz84OnpiTFjxiA/P99GFVvP22+/DYlEghkzZojHWltfL126hL/85S/w8/ODm5sboqOjcejQIfG8IAiYP38+goOD4ebmhri4OJw5c8aGFTedyWTCvHnzEB4eDjc3N3Ts2BFvvvmmxb2KHLm/u3fvxujRoxESEgKJRIJ169ZZnG9M30pKSpCQkACVSgVvb29MnjwZFRUVLdiLxrtZf41GI+bOnYvo6Gh4eHggJCQEEydORG5ursV7tJb+/tGzzz4LiUSC999/3+K4o/S3MX1NT0/Hgw8+CLVaDQ8PD/Tr1w/Z2dni+Zb8Xc1A5CR27dqFxMRE7N+/H0lJSTAajRg2bBgqKyvFNjNnzsTPP/+MH3/8Ebt27UJubi4effRRG1Z95w4ePIh///vf6NGjh8Xx1tTX0tJSDBw4EC4uLti0aRNOnjyJf/3rX/Dx8RHbLF68GMuWLcOqVauQkpICDw8PxMfHo6amxoaVN82iRYuwcuVKfPDBB0hPT8eiRYuwePFiLF++XGzjyP2trKxEz549sWLFiuueb0zfEhIScOLECSQlJWHDhg3YvXs3pkyZ0lJduC03629VVRUOHz6MefPm4fDhw1izZg0yMjLw4IMPWrRrLf292tq1a7F//36EhIRcc85R+nurvp47dw6DBg1CREQEdu7ciaNHj2LevHlwdXUV27To72qBnFJBQYEAQNi1a5cgCIJQVlYmuLi4CD/++KPYJj09XQAgJCcn26rMO1JeXi507txZSEpKEu69917h+eefFwSh9fV17ty5wqBBg2543mw2C0FBQcI777wjHisrKxOUSqXwn//8pyVKtKpRo0YJf/vb3yyOPfroo0JCQoIgCK2rvwCEtWvXis8b07eTJ08KAISDBw+KbTZt2iRIJBLh0qVLLVZ7U/yxv9dz4MABAYCQlZUlCELr7O/FixeFNm3aCMePHxfatWsnvPfee+I5R+3v9fr6xBNPCH/5y19u+JqW/l3NESInpdPpAAC+vr4AgNTUVBiNRsTFxYltIiIiEBYWhuTkZJvUeKcSExMxatQoiz4Bra+v69evR0xMDB577DEEBgaid+/e+Pjjj8XzmZmZ0Gq1Fv1Vq9Xo37+/Q/Z3wIAB2LZtG06fPg0AOHLkCPbs2YMRI0YAaH39vVpj+pacnAxvb2/ExMSIbeLi4iCVSpGSktLiNVubTqeDRCKBt7c3gNbXX7PZjAkTJmDOnDno1q3bNedbS3/NZjM2btyILl26ID4+HoGBgejfv7/FtFpL/65mIHJCZrMZM2bMwMCBA9G9e3cAgFarhUKhEH/JNNBoNNBqtTao8s589913OHz4MBYuXHjNudbW1/Pnz2PlypXo3LkztmzZgqlTp2L69On48ssvAUDsk0ajsXido/b3pZdewtixYxEREQEXFxf07t0bM2bMQEJCAoDW19+rNaZvWq0WgYGBFuflcjl8fX0dvv81NTWYO3cuxo0bJ94AtLX1d9GiRZDL5Zg+ffp1z7eW/hYUFKCiogJvv/02hg8fjq1bt+KRRx7Bo48+il27dgFo+d/VvNu9E0pMTMTx48exZ88eW5fSLHJycvD8888jKSnJYi66tTKbzYiJicGCBQsAAL1798bx48exatUqTJo0ycbVWd8PP/yA1atX49tvv0W3bt2QlpaGGTNmICQkpFX2l+oZjUY8/vjjEAQBK1eutHU5zSI1NRVLly7F4cOHIZFIbF1OszKbzQCAhx56CDNnzgQA9OrVC/v27cOqVatw7733tnhNHCFyMtOmTcOGDRuwY8cOtG3bVjweFBQEg8GAsrIyi/b5+fkICgpq4SrvTGpqKgoKCtCnTx/I5XLI5XLs2rULy5Ytg1wuh0ajaTV9BYDg4GBERUVZHIuMjBSv1Gjo0x+vzHDU/s6ZM0ccJYqOjsaECRMwc+ZMcTSwtfX3ao3pW1BQEAoKCizO19XVoaSkxGH73xCGsrKykJSUJI4OAa2rv7/99hsKCgoQFhYm/u7KysrCCy+8gPbt2wNoPf319/eHXC6/5e+ulvxdzUDkJARBwLRp07B27Vps374d4eHhFuf79u0LFxcXbNu2TTyWkZGB7OxsxMbGtnS5d2To0KE4duwY0tLSxEdMTAwSEhLEP7eWvgLAwIEDr9lC4fTp02jXrh0AIDw8HEFBQRb91ev1SElJccj+VlVVQSq1/NUlk8nE/+Jsbf29WmP6Fhsbi7KyMqSmpopttm/fDrPZjP79+7d4zXeqIQydOXMGv/76K/z8/CzOt6b+TpgwAUePHrX43RUSEoI5c+Zgy5YtAFpPfxUKBfr163fT310t/r1k9WXaZJemTp0qqNVqYefOnUJeXp74qKqqEts8++yzQlhYmLB9+3bh0KFDQmxsrBAbG2vDqq3n6qvMBKF19fXAgQOCXC4X/vnPfwpnzpwRVq9eLbi7uwvffPON2Obtt98WvL29hZ9++kk4evSo8NBDDwnh4eFCdXW1DStvmkmTJglt2rQRNmzYIGRmZgpr1qwR/P39hRdffFFs48j9LS8vF37//Xfh999/FwAIS5YsEX7//XfxqqrG9G348OFC7969hZSUFGHPnj1C586dhXHjxtmqSzd1s/4aDAbhwQcfFNq2bSukpaVZ/O6qra0V36O19Pd6/niVmSA4Tn9v1dc1a9YILi4uwkcffSScOXNGWL58uSCTyYTffvtNfI+W/F3NQOQkAFz38fnnn4ttqqurhb///e+Cj4+P4O7uLjzyyCNCXl6e7Yq2oj8GotbW159//lno3r27oFQqhYiICOGjjz6yOG82m4V58+YJGo1GUCqVwtChQ4WMjAwbVXtn9Hq98PzzzwthYWGCq6ur0KFDB+GVV16x+IJ05P7u2LHjuv9WJ02aJAhC4/pWXFwsjBs3TvD09BRUKpXw17/+VSgvL7dBb27tZv3NzMy84e+uHTt2iO/RWvp7PdcLRI7S38b09dNPPxU6deokuLq6Cj179hTWrVtn8R4t+btaIghXbe9KRERE5IS4hoiIiIicHgMREREROT0GIiIiInJ6DERERETk9BiIiIiIyOkxEBEREZHTYyAiIiIip8dARERERE6PgYiI7Fb79u3x/vvvN7r9zp07IZFIrrkZJBHRrXCnaiKymvvuuw+9evW6rRBzM4WFhfDw8IC7u3uj2hsMBpSUlECj0UAikVilhtu1c+dODBkyBKWlpfD29rZJDUR0++S2LoCInIsgCDCZTJDLb/3rJyAg4LbeW6FQICgoqKmlEZET45QZEVnFk08+iV27dmHp0qWQSCSQSCS4cOGCOI21adMm9O3bF0qlEnv27MG5c+fw0EMPQaPRwNPTE/369cOvv/5q8Z5/nDKTSCT45JNP8Mgjj8Dd3R2dO3fG+vXrxfN/nDL74osv4O3tjS1btiAyMhKenp4YPnw48vLyxNfU1dVh+vTp8Pb2hp+fH+bOnYtJkybh4YcfvmFfs7KyMHr0aPj4+MDDwwPdunXDL7/8ggsXLmDIkCEAAB8fH0gkEjz55JMAALPZjIULFyI8PBxubm7o2bMn/vvf/15T+8aNG9GjRw+4urri7rvvxvHjx2/5uUR05xiIiMgqli5ditjYWDz99NPIy8tDXl4eQkNDxfMvvfQS3n77baSnp6NHjx6oqKjAyJEjsW3bNvz+++8YPnw4Ro8ejezs7Jt+zuuvv47HH38cR48exciRI5GQkICSkpIbtq+qqsK7776Lr7/+Grt370Z2djZmz54tnl+0aBFWr16Nzz//HHv37oVer8e6detuWkNiYiJqa2uxe/duHDt2DIsWLYKnpydCQ0Pxv//9DwCQkZGBvLw8LF26FACwcOFCfPXVV1i1ahVOnDiBmTNn4i9/+Qt27dpl8d5z5szBv/71Lxw8eBABAQEYPXo0jEbjTT+XiKxAICKyknvvvVd4/vnnLY7t2LFDACCsW7fulq/v1q2bsHz5cvF5u3bthPfee098DkB49dVXxecVFRUCAGHTpk0Wn1VaWioIgiB8/vnnAgDh7Nmz4mtWrFghaDQa8blGoxHeeecd8XldXZ0QFhYmPPTQQzesMzo6Wnjttdeue+6PNQiCINTU1Aju7u7Cvn37LNpOnjxZGDdunMXrvvvuO/F8cXGx4ObmJnz//fe3/FwiujNcQ0RELSImJsbieUVFBV577TVs3LgReXl5qKurQ3V19S1HiHr06CH+2cPDAyqVCgUFBTds7+7ujo4dO4rPg4ODxfY6nQ75+fm46667xPMymQx9+/aF2Wy+4XtOnz4dU6dOxdatWxEXF4cxY8ZY1PVHZ8+eRVVVFe6//36L4waDAb1797Y4FhsbK/7Z19cXXbt2RXp6epM+l4gaj1NmRNQiPDw8LJ7Pnj0ba9euxYIFC/Dbb78hLS0N0dHRMBgMN30fFxcXi+cSieSm4eV67YU7vLj2qaeewvnz5zFhwgQcO3YMMTExWL58+Q3bV1RUAAA2btyItLQ08XHy5EmLdUTW/lwiajwGIiKyGoVCAZPJ1Ki2e/fuxZNPPolHHnkE0dHRCAoKwoULF5q3wD9Qq9XQaDQ4ePCgeMxkMuHw4cO3fG1oaCieffZZrFmzBi+88AI+/vhjAPU/g4b3aRAVFQWlUons7Gx06tTJ4nH1OisA2L9/v/jn0tJSnD59GpGRkbf8XCK6M5wyIyKrad++PVJSUnDhwgV4enrC19f3hm07d+6MNWvWYPTo0ZBIJJg3b95NR3qay3PPPYeFCxeiU6dOiIiIwPLly1FaWnrTfYxmzJiBESNGoEuXLigtLcWOHTvE0NKuXTtIJBJs2LABI0eOhJubG7y8vDB79mzMnDkTZrMZgwYNgk6nw969e6FSqTBp0iTxvd944w34+flBo9HglVdegb+/v3jF280+l4juDEeIiMhqZs+eDZlMhqioKAQEBNx0PdCSJUvg4+ODAQMGYPTo0YiPj0efPn1asNp6c+fOxbhx4zBx4kTExsbC09MT8fHxcHV1veFrTCYTEhMTERkZieHDh6NLly748MMPAQBt2rTB66+/jpdeegkajQbTpk0DALz55puYN28eFi5cKL5u48aNCA8Pt3jvt99+G88//zz69u0LrVaLn3/+2WLU6UafS0R3hjtVExFdxWw2IzIyEo8//jjefPPNFvtc7nBNZFucMiMip5aVlYWtW7fi3nvvRW1tLT744ANkZmZi/Pjxti6NiFoQp8yIyKlJpVJ88cUX6NevHwYOHIhjx47h119/5docIifDKTMiIiJyehwhIiIiIqfHQEREREROj4GIiIiInB4DERERETk9BiIiIiJyegxERERE5PQYiIiIiMjpMRARERGR0/t/o2WR63SmeAoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5476059438635112, recall = 0.23724368144969005, f1 = 0.3310597238396273\n",
      "precision = 0.49768875192604006, recall = 0.19779546846295162, f1 = 0.2830850131463628\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ab41a9f0",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=32.62: 100%|█| 20/20 [02:08<00:00,  6.44s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAb+BJREFUeJzt3Xl4lNX9/vH3zGTfF7IQSNi3sAsIEVRUlEURFeuGa12qX6gV19pq6/KrWLVuaLVa69JKa1XQCqKy7ztE9j2QAFmAkH2feX5/zMxDBhIIIWSScL+uK1cz8zwzcw5IcveczznHYhiGgYiIiEgLZfV2A0RERETOJYUdERERadEUdkRERKRFU9gRERGRFk1hR0RERFo0hR0RERFp0RR2REREpEXz8XYDmgKHw8GhQ4cIDQ3FYrF4uzkiIiJSB4ZhUFhYSEJCAlZr7eM3CjvAoUOHSExM9HYzREREpB4yMjJo27ZtrdcVdoDQ0FDA+YcVFhbm5daIiIhIXRQUFJCYmGj+Hq+Nwg6YU1dhYWEKOyIiIs3M6UpQVKAsIiIiLZrCjoiIiLRoXg077733Hn369DGnj1JSUpg9e7Z5vaysjIkTJxIdHU1ISAjjx48nOzvb4z3S09O5+uqrCQoKIjY2lieeeIKqqqrG7oqIiIg0UV6t2Wnbti0vv/wyXbp0wTAMPv30U8aNG8eGDRvo2bMnkydPZtasWXz55ZeEh4czadIkbrjhBpYtWwaA3W7n6quvJj4+nuXLl5OZmcmdd96Jr68vL730kje7JiIiTYjdbqeystLbzZAz5Ovri81mO+v3sRiGYTRAexpMVFQUr776KjfeeCMxMTFMmzaNG2+8EYDt27fTo0cPVqxYwZAhQ5g9ezbXXHMNhw4dIi4uDoD333+fp556isOHD+Pn51enzywoKCA8PJz8/HwVKIuItCCGYZCVlUVeXp63myL1FBERQXx8fI1FyHX9/d1kVmPZ7Xa+/PJLiouLSUlJYd26dVRWVjJixAjznu7du5OUlGSGnRUrVtC7d28z6ACMHDmShx56iC1bttC/f/8aP6u8vJzy8nLzcUFBwbnrmIiIeI076MTGxhIUFKSNY5sRwzAoKSkhJycHgNatW9f7vbwedjZt2kRKSgplZWWEhIQwY8YMkpOTSU1Nxc/Pj4iICI/74+LiyMrKApz/EVcPOu7r7mu1mTJlCs8//3zDdkRERJoUu91uBp3o6GhvN0fqITAwEICcnBxiY2PrPaXl9dVY3bp1IzU1lVWrVvHQQw9x1113sXXr1nP6mU8//TT5+fnmV0ZGxjn9PBERaXzuGp2goCAvt0TOhvvv72xqrrw+suPn50fnzp0BGDBgAGvWrOGtt97i5ptvpqKigry8PI/RnezsbOLj4wGIj49n9erVHu/nXq3lvqcm/v7++Pv7N3BPRESkKdLUVfPWEH9/Xh/ZOZHD4aC8vJwBAwbg6+vLvHnzzGs7duwgPT2dlJQUAFJSUti0aZM5nwcwZ84cwsLCSE5ObvS2i4iISNPj1ZGdp59+mtGjR5OUlERhYSHTpk1j4cKF/Pjjj4SHh3Pvvffy6KOPEhUVRVhYGL/+9a9JSUlhyJAhAFx11VUkJydzxx138Morr5CVlcUzzzzDxIkTNXIjIiIigJdHdnJycrjzzjvp1q0bV1xxBWvWrOHHH3/kyiuvBOCNN97gmmuuYfz48VxyySXEx8czffp08/U2m42ZM2dis9lISUnh9ttv58477+SFF17wVpdERESanPbt2/Pmm296/T28xasjOx999NEprwcEBPDuu+/y7rvv1npPu3bt+P777xu6aY2myu7Abhj4+5z9pkkiItIyDB8+nH79+jVYuFizZg3BwcEN8l7NUZOr2TmfVNod3Pb3VVz4p3nkFJR5uzkiItKMGIZR5+ORYmJizutVaQo7XvTO/N2sTsslv7SSH7bUvi+QiIg0DMMwKKmo8spXXQ8suPvuu1m0aBFvvfUWFosFi8XCvn37WLhwIRaLhdmzZzNgwAD8/f1ZunQpe/bsYdy4ccTFxRESEsKgQYOYO3eux3ueOAVlsVj4+9//zvXXX09QUBBdunThf//73xn9WaanpzNu3DhCQkIICwvjpptu8ji/8ueff+ayyy4jNDSUsLAwBgwYwNq1awHYv38/Y8eOJTIykuDgYHr27HlOZ2m8vvT8fLXpQD7vLNhtPp6zNZs7U9p7r0EiIueB0ko7yX/40SufvfWFkQT5nf7X7ltvvcXOnTvp1auXWYMaExPDvn37APjtb3/La6+9RseOHYmMjCQjI4MxY8bwpz/9CX9/fz777DPGjh3Ljh07SEpKqvVznn/+eV555RVeffVVpk6dyoQJE9i/fz9RUVGnbaPD4TCDzqJFi6iqqmLixIncfPPNLFy4EIAJEybQv39/3nvvPWw2G6mpqfj6+gIwceJEKioqWLx4McHBwWzdupWQkJDTfm59Kex4QVmlnUf/m4rdYTCwXSRr9x9j5d6jFJZVEhrg6+3miYiIF4WHh+Pn50dQUFCNe8a98MIL5kIecJ4p2bdvX/Pxiy++yIwZM/jf//7HpEmTav2cu+++m1tvvRWAl156ibfffpvVq1czatSo07Zx3rx5bNq0ibS0NBITEwH47LPP6NmzJ2vWrGHQoEGkp6fzxBNP0L17dwC6dOlivj49PZ3x48fTu3dvADp27HjazzwbCjte8MbcnezKKaJViD8f3DmQG99fzt7DxSzeeYSr+9T/7A8RETm1QF8bW18Y6bXPbggDBw70eFxUVMRzzz3HrFmzyMzMpKqqitLSUtLT00/5Pn369DG/Dw4OJiwszGPfulPZtm0biYmJZtABSE5OJiIigm3btjFo0CAeffRR7rvvPv75z38yYsQIfvGLX9CpUycAHn74YR566CF++uknRowYwfjx4z3a09BUs+MFX687AMCL43oSFezHlT2c53nN3ZZ9qpeJiMhZslgsBPn5eOWroXZyPnFV1eOPP86MGTN46aWXWLJkCampqfTu3ZuKiopTvo97Sqn6n43D4WiQNgI899xzbNmyhauvvpr58+eTnJzMjBkzALjvvvvYu3cvd9xxB5s2bWLgwIFMnTq1wT77RAo7jaygrJIjRc7/AC/uGgPAFa6wM397DlX2hvsPTUREmic/Pz/sdnud7l22bBl33303119/Pb179yY+Pt6s7zlXevToQUZGhsfZklu3biUvL8/jBIOuXbsyefJkfvrpJ2644QY+/vhj81piYiIPPvgg06dP57HHHuPDDz88Z+1V2Glk+44UA9AqxJ8Qf+cs4gVJEUQG+ZJfWsna/ce82TwREWkC2rdvz6pVq9i3bx9Hjhw55YhLly5dmD59Oqmpqfz888/cdtttDTpCU5MRI0bQu3dvJkyYwPr161m9ejV33nknl156KQMHDqS0tJRJkyaxcOFC9u/fz7Jly1izZg09evQA4JFHHuHHH38kLS2N9evXs2DBAvPauaCw08jSXGGnY6vjw5A+NiuXdY8FYO5WTWWJiJzvHn/8cWw2G8nJycTExJyy/ub1118nMjKSiy66iLFjxzJy5EguuOCCc9o+i8XCt99+S2RkJJdccgkjRoygY8eOfPHFF4DzhIOjR49y55130rVrV2666SZGjx7N888/D4DdbmfixIn06NGDUaNG0bVrV/7617+eu/YadV3434IVFBQQHh5Ofn4+YWFh5/Sz3py7kzfn7uKmgW155cbj1fOzN2Xy0OfraRcdxMLHh+uUXhGRs1RWVkZaWhodOnQgICDA282RejrV32Ndf39rZKeRuaex2rfyLDC7uGsMfjYr+4+WkJ5b4o2miYiItEgKO40s7agzyHSI9gw7If4+tI5wJtYjReWN3i4REZGWSmGnkdU2sgOYBcsFZXU760REREROT2GnER0rriC/tBKA9tG1h50ihR0RkQaj0tTmrSH+/hR2GtFe16hO6/AAAv1O3knTfVREUbnCjojI2XJvmldSojrI5sz993fiJohnQsdFNCJzCquGUR2A0ADnX0dhWWWjtUlEpKWy2WxERESYRyAEBQVppWszYhgGJSUl5OTkEBERgc1W/+M2FHYa0b6jtdfrgKaxREQamvsgzbqe+SRNT0RERI0Hop4JhZ1GVNOGgtWFuEd2NI0lItIgLBYLrVu3JjY2lspKjZo3N76+vmc1ouOmsNOI0k6xEguOT2NpZEdEpGHZbLYG+aUpzZMKlBuJYRhmzU6HVkE13hPq767ZUdgRERFpKAo7jeRwUTnFFXasFkiMqjnsuKextBpLRESk4SjsNJJ9R5xL5xIiAvH3qXkoNdTfuaxONTsiIiINR2HnHDpwrIRDeaUA1aawaq7XgWojO1p6LiIi0mBUoHyOGIbBU19vJDU9j8eu6kZWQRlwmrCjmh0REZEGp7BzjhSUVVFW6aC4ws4LM7dide1jVduGglBtNZamsURERBqMprHOkfBAX778VQp/ur4XoQE+OFxHe5xqZMd9XERJhR27Q2e5iIiINASN7JxDVquFCYPbcWVyHC/P3k5GbgkXdoiq9f5g/+OFy0VlVYQH1f8cEBEREXFS2GkEsaEBvH5Tv9Pe5+9jw8/HSkWVg8LySoUdERGRBqBprCYmTHU7IiIiDUphp4nRYaAiIiINS2GniTEPA1XYERERaRAKO02MudeOprFEREQahMJOE+Nefq5pLBERkYahsNPEuE8+LyrXkREiIiINQWGniVHNjoiISMNS2GliQhV2REREGpTCThMT4u+q2VGBsoiISINQ2Glijk9jqWZHRESkISjsNDHHC5Q1siMiItIQFHaaGHfNjpaei4iINAyFnSZGmwqKiIg0LIWdJkZLz0VERBqWwk4TE6YdlEVERBqUwk4T457GKq20U2V3eLk1IiIizZ/CThMT7Ao7AMXldi+2REREpGVQ2Gli/Hys+Ps4/1oKtNeOiIjIWVPYaYLMk8+1IktEROSsKew0QeZeOwo7IiIiZ01hpwky99rRNJaIiMhZU9hpgnTyuYiISMNR2GmCQk44H+vtebuYOG09dofhzWaJiIg0Swo7TVBItfOxSiqqeGveLmZtzGRndqGXWyYiItL8KOw0QaH+x6exfs7IN0d08kpUwyMiInKmvBp2pkyZwqBBgwgNDSU2NpbrrruOHTt2eNwzfPhwLBaLx9eDDz7ocU96ejpXX301QUFBxMbG8sQTT1BV1XzrXaovPV+3P9d8Pr+0wltNEhERabZ8Tn/LubNo0SImTpzIoEGDqKqq4ne/+x1XXXUVW7duJTg42Lzv/vvv54UXXjAfBwUFmd/b7Xauvvpq4uPjWb58OZmZmdx55534+vry0ksvNWp/Gkr1w0D3Hy02n9fIjoiIyJnzatj54YcfPB5/8sknxMbGsm7dOi655BLz+aCgIOLj42t8j59++omtW7cyd+5c4uLi6NevHy+++CJPPfUUzz33HH5+fue0D+eCu0C5oKySdfuPmc/nlyrsiIiInKkmVbOTn58PQFRUlMfzn3/+Oa1ataJXr148/fTTlJSUmNdWrFhB7969iYuLM58bOXIkBQUFbNmypcbPKS8vp6CgwOOrKXEvPf85I4+CasvP8xR2REREzphXR3aqczgcPPLIIwwdOpRevXqZz9922220a9eOhIQENm7cyFNPPcWOHTuYPn06AFlZWR5BBzAfZ2Vl1fhZU6ZM4fnnnz9HPTl77rCTU1ju8bymsURERM5ckwk7EydOZPPmzSxdutTj+QceeMD8vnfv3rRu3ZorrriCPXv20KlTp3p91tNPP82jjz5qPi4oKCAxMbF+DT8HQvx9PR5HBPmSV1KpAmUREZF6aBLTWJMmTWLmzJksWLCAtm3bnvLewYMHA7B7924A4uPjyc7O9rjH/bi2Oh9/f3/CwsI8vpoSd82O2/CuMYBGdkREROrDq2HHMAwmTZrEjBkzmD9/Ph06dDjta1JTUwFo3bo1ACkpKWzatImcnBzznjlz5hAWFkZycvI5afe55p7GcruseyygAmUREZH68Oo01sSJE5k2bRrffvstoaGhZo1NeHg4gYGB7Nmzh2nTpjFmzBiio6PZuHEjkydP5pJLLqFPnz4AXHXVVSQnJ3PHHXfwyiuvkJWVxTPPPMPEiRPx9/f3ZvfqrXrY6RoXQlKUc6m9RnZERETOnFdHdt577z3y8/MZPnw4rVu3Nr+++OILAPz8/Jg7dy5XXXUV3bt357HHHmP8+PF899135nvYbDZmzpyJzWYjJSWF22+/nTvvvNNjX57mJrjaNNaAdpGEBzpreDSyIyIicua8OrJjGKc+2DIxMZFFixad9n3atWvH999/31DN8jpfm5UAXytllQ4uSIokIsi5V1BReRWVdge+tiZRaiUiItIs6LdmE9U+Ohhfm4WUTtGEVZvW0uiOiIjImWkyS8/F0z/vHcyxkgraRjrrdUIDfCgsqyKvpJJWIc2zFklERMQbFHaaqJhQf2JCj4eaiCBfCsuqNLIjIiJyhjSN1UxEBDrrdrSxoIiIyJlR2Gkm3CuytPxcRETkzCjsNBPhQQo7IiIi9aGw00xEuEd2VLMjIiJyRhR2mokI18hOgcKOiIjIGVHYaSbcBcp5JSpQFhERORMKO81EuKaxRERE6kVhp5lQgbKIiEj9KOw0ExE6DFRERKReFHaaCfdhoAo7IiIiZ0Zhp5mIMKexKnA4Tn1avIiIiBynsNNMuAuUHQYUVVR5uTUiIiLNh8JOMxHga8Pfx/nXla8iZRERkTpT2GlGIrQiS0RE5Iwp7DQj5saCOvlcRESkzhR2mhH3XjtakSUiIlJ3CjvNiHkYqKaxRERE6kxhpxkJ18aCIiIiZ0xhpxmpvteOiIiI1I3CTjPi3kVZ01giIiJ1p7DTjGgaS0RE5Mwp7DQj5jSWwo6IiEidKew0I+59drSDsoiISN0p7DQj7mksbSooIiJSdwo7zYiOixARETlzCjvNiHsH5fIqB2WVdi+3RkREpHlQ2GlGQv19sFktgFZkiYiI1JXCTjNisViIdI3uHC4s93JrREREmgeFnWYmLiwAgOyCMi+3REREpHlQ2Glm4s2wo5EdERGRulDYaWbiwp1hJ0sjOyIiInWisNPMxIW6RnbyFXZERETqQmGnmYkP9wc0siMiIlJXCjvNjAqURUREzozCTjMTr5odERGRM6Kw08y4V2PllVRqF2UREZE6UNhpZsIDffH3cf615Wj5uYiIyGkp7DQzFotFU1kiIiJnQGGnGXIvP1fYEREROT2FnWbIvbGg9toRERE5PYWdZig+THvtiIiI1JXCTjOkvXZERETqTmGnGXIXKCvsiIiInJ7CTjPk3mtH01giIiKnp7DTDB2fxirHMAwvt0ZERKRpU9hphmJdBcoVVQ6OlVR6uTUiIiJNm8JOM+TvYyMq2A+ALC0/FxEROSWFnWbKnMoqVNgRERE5FYWdZsq91442FhQRETk1hZ1mSudjiYiI1I3CTjOljQVFRETqxqthZ8qUKQwaNIjQ0FBiY2O57rrr2LFjh8c9ZWVlTJw4kejoaEJCQhg/fjzZ2dke96Snp3P11VcTFBREbGwsTzzxBFVVVY3ZlUZn7rWjaSwREZFT8mrYWbRoERMnTmTlypXMmTOHyspKrrrqKoqLi817Jk+ezHfffceXX37JokWLOHToEDfccIN53W63c/XVV1NRUcHy5cv59NNP+eSTT/jDH/7gjS41mjhzGqvcyy0RERFp2ixGE9qV7vDhw8TGxrJo0SIuueQS8vPziYmJYdq0adx4440AbN++nR49erBixQqGDBnC7Nmzueaaazh06BBxcXEAvP/++zz11FMcPnwYPz+/kz6nvLyc8vLjIaGgoIDExETy8/MJCwtrnM6epa2HChjz9hKig/1Y9+yV3m6OiIhIoysoKCA8PPy0v7+bVM1Ofn4+AFFRUQCsW7eOyspKRowYYd7TvXt3kpKSWLFiBQArVqygd+/eZtABGDlyJAUFBWzZsqXGz5kyZQrh4eHmV2Ji4rnq0jnjLlA+WlxBeZXdy60RERFpuppM2HE4HDzyyCMMHTqUXr16AZCVlYWfnx8REREe98bFxZGVlWXeUz3ouK+7r9Xk6aefJj8/3/zKyMho4N6ce5FBvvj7OP/6MvNUtyMiIlIbH283wG3ixIls3ryZpUuXnvPP8vf3x9/f/5x/zrlksVjoGhfKpoP5bMssoH2rYG83SUREpElqEiM7kyZNYubMmSxYsIC2bduaz8fHx1NRUUFeXp7H/dnZ2cTHx5v3nLg6y/3YfU9L1TPBOT+5+VC+l1siIiLSdHk17BiGwaRJk5gxYwbz58+nQ4cOHtcHDBiAr68v8+bNM5/bsWMH6enppKSkAJCSksKmTZvIyckx75kzZw5hYWEkJyc3Tke8pGebcAA2HyzwcktERESaLq9OY02cOJFp06bx7bffEhoaatbYhIeHExgYSHh4OPfeey+PPvooUVFRhIWF8etf/5qUlBSGDBkCwFVXXUVycjJ33HEHr7zyCllZWTzzzDNMnDix2U9VnU4v18jOlkP5GIaBxWLxcotERESaHq+Gnffeew+A4cOHezz/8ccfc/fddwPwxhtvYLVaGT9+POXl5YwcOZK//vWv5r02m42ZM2fy0EMPkZKSQnBwMHfddRcvvPBCY3XDa7rHh2G1wJGiCnIKy81dlUVEROS4JrXPjrfUdZ1+U3TVG4vYmV3ER3cN5Ioecad/gYiISAvRLPfZkTPXM8FZt7PlkOp2REREaqKw08yZK7IOOldkGYbB72Zs4lf/XKvNBkVERFDYafZ6tfEc2Vm3/xjTVqXz45ZsPl62r9bXpWbksW5/bmM0UURExKsUdpq5ZNfIzsG8Uo4VV3gEnHfm7+Zw4ckHhf64JYvr/7qMG99fwdyt2SddFxERaUkUdpq5sABf2kUHATBnazazN2cC0C46iKLyKl77cYfH/ev2H+Phf2/AMMAw4JEvUtmdU9jo7RYREWksCjstgLtuZ8rsbTgMGNo5mtdv6gfAf9dlmPU8ew8Xcd+nayivcnB591gu7BBFUXkV93+2jvzSSm81X0RE5JxS2GkB3CuyjpU4A8svh3ZgQLtIxvVLwDDg7o9XM/iluVz+l0UcK6mkT9tw3rmtP3+dcAFtIgJJO1LMY/9N9WIPREREzh2FnRbAPbID0D46iMu6xQLw29HdCfKzcaSoguyCcvPej+4aRJCfD61C/PnbHQMAmLsth9ziisZvvIiIyDnWZE49l/pzj+wA3DO0A1ar89iI1uGBzPi/oezOKaJtZCCJUUFEBvl6HCvRq004MaH+HC4s5+CxUqKC/Rq9/SIiIueSwk4LEBPqz4gecRzKK2X8gLYe17rFh9ItPvSUr0+ICHSGnbxSercNP+W9IiIizY3CTgvx97sG1vu1bSMC+Tkjj4N5pQ3YIhERkaZBNTtCQoTzANFDCjsiItICKewICRGBQO1hp7i8irFTl3LnP1ajc2NFRKS50TSW0MYVdmqbxpo6fzebXHv1pOeW0C46uNHaJiIicrY0siOnHNnZe7iIj5buNR+v23+s0dolIiLSEBR2xBzZOVJUQVnl8ZPSDcPghZlbqbQbuFazs1ZhR0REmhmFHSEiyJcgPxvgObozb1sOC3ccxtdm4clR3QFYr7AjIiLNjMKOYLFYqk1llQFgdxi8OGsrAL8c1oHxFzj379mRXUhBmc7REhGR5kNhR4DjU1nukZ1dOYXsP1pCsJ+NX1/ehZhQf9pFB2EYsCE9z4stFREROTMKOwIcL1I+4Ao7GzOcq696tw0nxN+5aG9AUiSgImUREWleFHYEgDYnbCz484E8APq2jTDvGdDeGXZUtyMiIs2Jwo4A0CbScxpr4wHnyE6f6mGnnTPsbEg/RpXd0bgNFBERqSeFHQEgIfz4xoJllXa2ZxUA0KfawaBdYkMJ9fehuMLOjuxCr7RTRETkTCnsCHC8Ziczr4ytmQVU2g2igv1o6xrxAbBZLfRLigA0lSUiIs2Hwo4AEB8egNUCFXYH87flAM5RHYvF4nHfwHZRgDYXFBGR5kNhRwDwtVmJC3MWKc/enAl41uu4uet21u5T2BERkeZBYUdM7qmsPYeLAeiXGH7SPb1dNTwH80opLq9qvMaJiIjUk8KOmNwbC7rVNLITHuhLaIBz352aDg4VERFpahR2xJRQLey0iQikVYh/jfe1jQwC4MAxhR0REWn6FHbE5N5YEDyXnJ98n+duyyIiIk2Zwo6Y2lRbZl7TFJabezn6QY3siIhIM1CvsPPpp58ya9Ys8/GTTz5JREQEF110Efv372+wxknjqj6N1fcUIzvusHPgWEmt92Tml5JXUtFwjRMREamneoWdl156icBA5y+8FStW8O677/LKK6/QqlUrJk+e3KANlMbTNjIIX5sFP5uVXnWYxjpYyzTWobxSrnx9Mbd8sPKctFNERORM+NTnRRkZGXTu3BmAb775hvHjx/PAAw8wdOhQhg8f3pDtk0YU4u/D3+4YgM1qJSzAt9b72pxmGmv6+gMUlVexPauQsko7Ab62c9JeERGRuqjXyE5ISAhHjx4F4KeffuLKK68EICAggNJS1XE0Z5d3j+PSrjGnvMe9GiunsJzyKrvHNcMw+GrdAfNxVn5ZwzdSRETkDNRrZOfKK6/kvvvuo3///uzcuZMxY8YAsGXLFtq3b9+Q7ZMmKDLIl0BfG6WVdg7lldGhVbB5bd3+Y+w7eryWJ7ugjPbVrouIiDS2eo3svPvuu6SkpHD48GG+/vproqOjAVi3bh233nprgzZQmh6LxVLrVFb1UR2ArAKN7IiIiHfVa2QnIiKCd95556Tnn3/++bNukDQPbSMD2Z1TxMG846M4pRV2Zm50nqvVJiKQg3mlZCvsiIiIl9VrZOeHH35g6dKl5uN3332Xfv36cdttt3HsmA6IPB+YGwtWG9n5cUsWReVVtI0M5Oo+rQHIyi/3SvtERETc6hV2nnjiCQoKCgDYtGkTjz32GGPGjCEtLY1HH320QRsoTVNN01juKazxF7Ql3nWCukZ2RETE2+o1jZWWlkZycjIAX3/9Nddccw0vvfQS69evN4uVpWUzz8dy7bWTXVDGsj1HAGfY2XwoH1DNjoiIeF+9Rnb8/PwoKXHWasydO5errroKgKioKHPER1o2c2NB18jO3G3ZGAb0S4wgKTqIONfIjpaei4iIt9VrZGfYsGE8+uijDB06lNWrV/PFF18AsHPnTtq2bdugDZSmyX1kRFZBGVV2B3O3ZgNwZXIcAPHhzrCTU1iGw2FgtVq801ARETnv1Wtk55133sHHx4evvvqK9957jzZt2gAwe/ZsRo0a1aANlKYpJsQfP5sVu8Mg7Ugxy/Y4N5l0h53YUH8sFqi0G+TqjCwREfGieo3sJCUlMXPmzJOef+ONN866QdI8WK0WEiIC2He0hH+vzqCiykFSVBBdYkMA8LVZiQ7250hROVn5ZbQK8fdyi0VE5HxVr7ADYLfb+eabb9i2bRsAPXv25Nprr8Vm0zlI54s2kYHsO1rCl2szABjRIw6L5fh0VXy4M+xkF5TRq03tB4sahsF7i/bQPT6Uy7vHnfN2i4jI+aVeYWf37t2MGTOGgwcP0q1bNwCmTJlCYmIis2bNolOnTg3aSGma2kYEAUcpLK8CYERyrMf1+LAANh8sOO2KrO1Zhbzyww6ig/1Y+8wIj8AkIiJytupVs/Pwww/TqVMnMjIyWL9+PevXryc9PZ0OHTrw8MMPN3QbpYly77UDEBbgw6D2UR7X3Suysk+zIiun0Lnx4NHiCvN7ERGRhlKvkZ1FixaxcuVKoqKO/3KLjo7m5ZdfZujQoQ3WOGna3MvPAS7rHouvzTM7uzcWPN3ITl61AubtWYVmSBIREWkI9RrZ8ff3p7Cw8KTni4qK8PPzO+tGSfPQttrIzogeJ9faxIW7w86pR2tyi4+HnR1Z2qdJREQaVr3CzjXXXMMDDzzAqlWrMAwDwzBYuXIlDz74INdee21Dt1GaqPatgrFYwM9m5dJuMSddj6/jNNaxkkrz++1ZJ4doERGRs1Gvaay3336bu+66i5SUFHx9fQGorKxk3LhxvPnmmw3ZPmnC4sICePuW/oQF+hIW4HvSdffGgtmFdZ/G2qGwIyIiDaxeYSciIoJvv/2W3bt3m0vPe/ToQefOnRu0cdL0je2bUOs1d+1NXkklZZV2AnxtLN11hOgQP3q0DjPvqz6ysyuniCq7Ax9bvQYdRURETlLnsHO608wXLFhgfv/666/X6T0XL17Mq6++yrp168jMzGTGjBlcd9115vW7776bTz/91OM1I0eO5IcffjAf5+bm8utf/5rvvvsOq9XK+PHjeeuttwgJCalTG+TcCQvwIdDXRmmlneyCMo6VVHL7R6tIigpi8ZOXmfcdq1azU1HlYN/REjrH6u9PREQaRp3DzoYNG+p035nskVJcXEzfvn355S9/yQ033FDjPaNGjeLjjz82H/v7e+7EO2HCBDIzM5kzZw6VlZXcc889PPDAA0ybNq3O7ZBzw2KxEB8eQNqRYrLyy/h6/QEA0nNLqLQ7zNVbx044TmJHVqHCjoiINJg6h53qIzcNZfTo0YwePfqU9/j7+xMfH1/jtW3btvHDDz+wZs0aBg4cCMDUqVMZM2YMr732GgkJtU+xSOOIC/Mn7Ugxuw8X8d3PmebzucUVHtNcAN3jQ9meVciOrAKu7tPaK+0VEZGWp8kXRixcuJDY2Fi6devGQw89xNGjR81rK1asICIiwgw6ACNGjMBqtbJq1apa37O8vJyCggKPLzk33CuyPlqSRmml3Xz+cLXNA90jO0M6RgPHV2SVVdr53YxN/HdNRmM1V0REWqAmHXZGjRrFZ599xrx58/jzn//MokWLGD16NHa785dmVlYWsbGeRxT4+PgQFRVFVlZWre87ZcoUwsPDza/ExMRz2o/zmXuvnb1Hij2eP1zkDDtllXZKKpx/n0M6Ojep3JHtDDv/WrmfaavSefbbzeRXK2IWERE5E0067Nxyyy1ce+219O7dm+uuu46ZM2eyZs0aFi5ceFbv+/TTT5Ofn29+ZWRo5OBcia+2G7KfzUqfts4DQY+4RnbcU1hWCwx0HTeRnlvCseIK/rZ4LwDlVQ6z3kdERORMNemwc6KOHTvSqlUrdu/eDUB8fDw5OTke91RVVZGbm1trnQ8464DCwsI8vuTcqB52RvaKp0tsKHB8ZMc9hRUZ5EerEH9ahfhjGPDizK0cLizHXe8+bXU6hmE0buNFRKRFaFZh58CBAxw9epTWrZ3FqykpKeTl5bFu3Trznvnz5+NwOBg8eLC3minVuKexAG4dlEirUOdxIkcKnSHHHXYigpybEnaPd4ah6RsOAvDkyO4E+trYnVPEmn3HGq3dIiLScng17BQVFZGamkpqaioAaWlppKamkp6eTlFREU888QQrV65k3759zJs3j3HjxtG5c2dGjhwJODcyHDVqFPfffz+rV69m2bJlTJo0iVtuuUUrsZqIrnGhtArx44KkCIZ0jCYmxLl1gHtkxz2NFRnkDEHdXGEHnCu57hnanmtdGxdOW7X/pPc/XFjOxa/M58WZW89pP0REpPnyathZu3Yt/fv3p3///oBz48L+/fvzhz/8AZvNxsaNG7n22mvp2rUr9957LwMGDGDJkiUee+18/vnndO/enSuuuIIxY8YwbNgwPvjgA291SU4Q4u/D8t9ewb8fGILVaiEm1Pl3567ZcR8CGlFD2PnVJZ0I8LVx2+AkAL7fnOWxASHAwh05ZOSW8vmq/ZRX2RERETlRvY6LaCjDhw8/ZR3Gjz/+eNr3iIqK0gaCTZyfz/FM7R7ZOWKO7LhrdpzTWP0TI5z3hfpz64XOkNOnbTg9E8LYcqiAr9cf4L6LO5rvtzXTuW1AWaWD1PQ8BruWr4uIiLg1q5odaf5ahXpOY7nPxYoKdo7sdIkLZdp9g/nyVykE+tkA507M7tGdr9Z5rsraeuj4HknL9xxFRETkRAo70qhauUZ28koqqahyVCtQ9jPvuahzK9q3CvZ43cieztV127MKzT13DMMwR3YAVijsiIhIDRR2pFFFBPriY3WuJz9aXF6tQNn3lK9rFeJP++ggANZnOFdlHThWSmFZlbk8fUPGMUoqqs5Ry0VEpLlS2JFGZbVaiA45vvz8xALlU7mgXSQA6/c7w457VKdHfBhtIgKptBtani4iIidR2JFGZ67IKio3C5TdNTunMsAVdta5w46rXqdnQhgpnZyFycv3HGnw9oqISPOmsCONzl23c7iw3CxQPt00FhwPO6kZeVTZHebITnJCGEM7O8OO6nZEROREXl16Lucnd9jJKiijoMwZduoyjdUlNpRQfx8Ky6vYnlVojuwktw6jXbSzoHnzwXzySyoJr0N4EhGR84NGdqTRuaex9hwuwr3NUkQdwonNaqFfUgQAC7bncDCvFIAeCWHEhwfQMSYYhwGr0jS6IyIixynsSKNzj+zsyi4CINTfB19b3f5TdE9l/Xt1OgCJUYGEBTiD0kVm3Y7CjoiIHKewI42u+sgOQGQdipPd3GHnUH4Z4JzCckvp2AqAtftzG6SdIiLSMijsSKNr5Vp6Xl7lAOpWnOzWLzEC1zY9ACS3Dje/7xYfAsD+IyWnPIZERETOLwo70uhiQ/09HtelONktNMCXbvHHR3OSE45/3zbSuelgYXkV+aWVZ9lKERFpKRR2pNG5a3bczmRkB2BAuwjz++phJ8DXZgap9NyS+jdQRERaFIUdaXThgb742o7PRZ3JyA4cr9sJD/QlITzA41pSlHN0R2FHRETcFHak0VksFo/RnbrsnlzdFT3iGNwhigcu6YjFYvG4lugKOxm5pWffUBERaRG0qaB4RasQfzJdK6rOdBorLMCXL36VUuO1xLMY2XE4DLYcKmDxrsP8nJHHLRcmcnn3uDN+HxERaVoUdsQrYqoVKZ/pNNapJJkjO7WHnbJKO/4+Vo9RoX1Hirnlg5VkFZSZz208kM+Sp2LqvAeQiIg0TfopLl7hXn4OENmAYScxMhCAjGM1h51Ve4/S74Wf+N2MzR7Pf7E2g6yCMoL9bFyZHEdUsB9ZBWX8tCW7wdomIiLeobAjXlG9ZicyuOHOsUqKdo7sHDxWSpXd4XGtpKKKJ77aSFmlg6/XHSDfdQipYRj8sDkLgJfH9+HDOwcyYXASAJ+u2NdgbRMREe9Q2BGvqD6N1ZAjO3GhAfjZrFQ5DLMmyO2VH3aYtTwVdgezN2cCsCO7kLQjxfj5WLmseywAEwa3w2a1sDotl22u09VFRKR5UtgRr/AY2WnAsGO1WmjrnsqqVrezOi2XT5bvA2B4txgAvkk9CMDsTc5RnUu6xBDi7yxjiw8PYFTPeAA+0+iOiEizprAjXuEe2fH3sRLoZ2vQ9zaXn7vqdsoq7Tz51c8A3DwwkT9d3xuAlXtzOZRXak5hje4V7/E+d13UHoAZGw6SV1LRoG0UEZHGo7AjXtE5NgQ/Hyvdqx3k2VBO3Fhw1sZM9h0tIS7Mn99f04M2EYFc2CEKgLfn7WJHdiE+VgsjenguMx/UPpLu8aGUVTr479qMBm+niIg0DoUd8YpWIf4sefIypt03uMHfOzHKOY2V7tpYcO4254qqmwcmEhbgLIYe1y8BgP+scYaYlE7RhJ+w34/FYjELlRdsP9zg7RQRkcahsCNeExcWQLB/w2/1VH2vnfIqO4t3OoPKiOTjIzdX927tcWTF6F6ta3wv98jTgTwdPyEi0lwp7EiLk1gt7Kzcm0txhZ24MH96JYSb90QE+XFpV+fKK6sFrupZ807JbSKco0SZeWXYHYb5/NGicu77dC1LdmnER0SkqVPYkRbHHXaOFlfw7QbniqsresRhtXqeo3XzoEQAhnWJOekkdre4sAB8rBaqHAbZ1XZXnr7+IHO3ZfPKDztqbUd5lZ1vUw9ytKj8rPojIiJnR2FHWpywAF8iXPU33208BMCVPU4eubkyOY4vH0zhrZv71fpeNquFBNfozoFjxw8X3Xe0GIDNh/LJLa55pdaM9Qf5zX9See2nnfXqh4iINAyFHWmR3HU7lXaDQF8bKZ2ia7xvUPsoIk9z6rp7356D1ep29h91fm8YsGz3kRpftzunCIDtWdqUUETEmxR2pEVKjAwyv7+4SysCfOu/l4+7budA7skjO0CtdTvuHZzdwUhERLxDYUdaJHfdDniuwqqPtq7g5J7GKq+ycyjvePBZuusIhmGc9LrMfOc9ucUVFJRVnlUbRESk/hR2pEVyT2NZLHC567yr+jo+jeUMLweOleIwINDXhp+PlUP5Zew5XHzS66qfzZWu0R0REa9R2JEWqV9iBBaL87yr2lZa1ZU77BxwHT+x3zWF1b5VMBe2d+7EfOJUVpXdQU7h8VVY1ae9qrM7DH7OyKO8yn5WbRQRkdop7EiLlJwQxk+PXMLU2/qf9Xu1qTay43AY7DviDD3to4MY1qUV4JzKqu5wUbnHvjy11e18tS6Dce8uY+q83WfdThERqZnCjrRYXeJCzeMhzkZ8WAA2q4VKu0FOYbk5stMuOpiLXWFnxd6jVFQ5zNccyivzeI/9tYzsbD5YYL5eRETODYUdkdPwsVlpHR4AOJef7zt6fGSnR3wY0cF+lFTYWZ9+zHxNVr5n2NlXy8iOu9B5W2aBx0iQiIg0HIUdkTpoU21jwX3VanasVos5lVW9bse9EstdKF3byI676Lmkwl5rXY+IiJwdhR2ROnAvP993pMRcgt4+OhiAi1wbFm5IzzPvd09jDenoLGDOLiintOLkIuSD1ZawbzmkzQdFRM4FhR2ROnCvyFqVdhS7wyDA10psqHOVVw/Xyeg7sgrN+7MKSs1rYQHOk93Tcz2nsgrKKiksqzIfbzmU73E9v7Syxv17RETkzCjsiNSBO+ys3e+sy2kXFWweLNolNhSLxXnw6GHXcnP3yE7r8EDat3KOAJ04TXWw2llbAFurjex8m3qQvs//xH/WZJyD3oiInF8UdkTqwL383L3iql308R2aA/1stHPV5uzMdo7uuAuUW4cH0M413XVi3Y67ONnfx/nPcMuhAnMk518r9wO1n7slIiJ1p7AjUgfVz9oCzNEat27xoQBszyp0bSjoCjsRAbSPdhcpe05juet1hnSMxma1kFtcQWZ+GYcLy80RpOo1PSIiUj8KOyJ1EB8egGvWCvAc2QHoFu+u2ykgu7AchwG+Ngutgv2rrciqOex0jAmmc0wI4BzdmbstG3epzolTXSIicuYUdkTqwNdmpXV4oPnYvRLLrVucc2RnR1YhWa5l53FhAVitltPW7LSJCKRngjMsbTmUz49bssx7cgrLdZSEiMhZUtgRqSP3XjtQ08iOM+zszC4yl6YnuMKR+95DeaUeuyy7R3baRASS7Ao7q/bmsny3czdli2skKfOE3ZhFROTMKOyI1JF7RZbfCaM84NxN2c/HSmmlndVpuYBz6gsgJsSfID8bDuP4YaJwvEC5TWQgvdqEA65jJ+wOOsYE09E1IqS6HRGRs6OwI1JH7rCTGBWIrXoBD84jJbrEOutuFu5w7qTcOsIZdiwWy0l1OxVVx09FT6g2suM2smc8bVxF0arbERE5Owo7InXUyRVm3FNWJ3LX7bhHYhJqqPFx1+1k5ZdhGM5l59HBfoQF+JqBCFxhxxWWDmhkR0TkrPh4uwEizcXoXq2p/IVhHg9xohNDkHsaC6BdK/dxE86wcyDPOcLTJiIQi6s4p2dCGOm5JcSHBdCnTThLdznDkkZ2RETOjkZ2ROrIz8fKjQPakhARWOP1E8NO9ZGdZNeREiv2OouPzZVYkcfvcYeocf0TsFot5rVDGtkRETkrGtkRaSDd4z3rbtw1OwDDu8Zis1rYmV3E/qPF5nES1Vd43Ta4HV3iQhnQLtJ1zVWzUy3sHMwrZeq8XUy8rDOJUZ4rwkREpGYa2RFpIHFh/uahn342K1FBfua18CBfLmzvPAF9ztZsDrqmsaqPEtmsFoZ0jMbX5vxn6R7ZycwvxeFw7jL4zvxd/GdNBn9fsvfcd0hEpIVQ2BFpIBaLxRzdiQ8PMA8KdbsyOQ5whp2aRnZOFBfqj81qodJumCu31rmOkdh7pLjW14mIiCevhp3FixczduxYEhISsFgsfPPNNx7XDcPgD3/4A61btyYwMJARI0awa9cuj3tyc3OZMGECYWFhREREcO+991JUVNSIvRA5zl23U7042c0ddtbsy2V7lvPA0Nrqf8C5nD0+zPk+B/NKyC+tZGe287/tvYcVdkRE6sqrYae4uJi+ffvy7rvv1nj9lVde4e233+b9999n1apVBAcHM3LkSMrKju8oO2HCBLZs2cKcOXOYOXMmixcv5oEHHmisLoh4GNjeWW/jLkiuLjEqiO7xoTgMOFLkHKlpG1l72IHjU1kHjpWyIf2Y+fyh/FLKKnWMhIhIXXi1QHn06NGMHj26xmuGYfDmm2/yzDPPMG7cOAA+++wz4uLi+Oabb7jlllvYtm0bP/zwA2vWrGHgwIEATJ06lTFjxvDaa6+RkJDQaH0RARjbJ8F11lV4jdevTI4zR3UslppHgKprGxHIapyFyXuqjeYYBqTnltDVtbfPgu05PPvtZp65OplRveI93qOwrJLQAN+z6JWISPPWZGt20tLSyMrKYsSIEeZz4eHhDB48mBUrVgCwYsUKIiIizKADMGLECKxWK6tWrar1vcvLyykoKPD4EmkIVquFge2jCPSz1XjdPZUFEBcaYBYj18Y9snPwWCnr9x/zuFZ9KuvLdRkcOFbKo/9NZXeOM0w5HAa/m7GJ3s/9xB+/3UyV3UFN/rs2gz7P/WgecyEi0tI02bCTleU8+TkuLs7j+bi4OPNaVlYWsbGxHtd9fHyIiooy76nJlClTCA8PN78SExMbuPUiNeuVEE5cmD/gucdObdwFzOm5JaRm5AHQNc65k3P1U9S3HHIG9pIKO//3+XpKKqp4/rstTFuVDsCnK/bzq3+uo7i86qTPmLkxk4KyKv6xNK3+HQPySysVmESkSWqyYedcevrpp8nPzze/MjIyvN0kOU9YrRZG9HAG+FOtxHJzB6I1+3IpKq8i2M/GyJ7Oaao018hOQVmleeZWdLAfO7OLGP3WEj5dsR+Au1La4e9jZd72HG7+YAV5JRUen7HfFZoW7MihqIYwVFe/m7GJm/62gh+31P5/NEREvKHJhp34eOcP9OzsbI/ns7OzzWvx8fHk5OR4XK+qqiI3N9e8pyb+/v6EhYV5fIk0lomXdeaaPq25/+KOp73XHYjKKp1TUP2TIunsOqMrzbX8fJtrVCchPIB3brsAq+X4gaP/77pePD+uF/9+YAjRwX5sPljA567RHoBKu4MDrt2cy6sczNvm+e+trsqr7Mzf5vy3+ONmhR0RaVqabNjp0KED8fHxzJs3z3yuoKCAVatWkZKSAkBKSgp5eXmsW7fOvGf+/Pk4HA4GDx7c6G0WqYuEiEDeue0CeretuYj5xHuruyApwjxUNM01IuOewkpOCCelUzTPXJ1MaIAPfxybzO1D2rleF8kDlzjD1bbM4zVqGbkl2F0bFgLM2phZrz6t359HqWt12OJdR8xNEEVEmgKvrsYqKipi9+7d5uO0tDRSU1OJiooiKSmJRx55hP/3//4fXbp0oUOHDjz77LMkJCRw3XXXAdCjRw9GjRrF/fffz/vvv09lZSWTJk3illtu0UosaRECfG20CvHjSJFz6umCdpG0b+UMO4cLyyksqzTDTs8E5wjlL4d14O6L2p+0qWEXV63P7pzj+1C5R4BC/X0oLK9i4c7DFJVXEeJ/Zj8alu4+bH5/pKicbVkFta5IExFpbF4d2Vm7di39+/enf//+ADz66KP079+fP/zhDwA8+eST/PrXv+aBBx5g0KBBFBUV8cMPPxAQcHy57ueff0737t254oorGDNmDMOGDeODDz7wSn9EzoXqtT39EyMJD/SlVYjzKIr9R0vYmukZdoCTgg5Al1jnMvW9R4rNlVnuqbBhXVrRsVUwFfWcylqy6wgAAb7OHymLdx454/cQETlXvBp2hg8fjmEYJ3198skngHP7/RdeeIGsrCzKysqYO3cuXbt29XiPqKgopk2bRmFhIfn5+fzjH/8gJCTEC70ROTfcRcpdYkMID3Lul+OeytqRVciubOdS855tTj2S0iYikABfKxVVDjJcdTruFV3tWwVzdZ/WgHN11pk4VlzBpoP5ANw7rAMAi3bmnOolIiKNqsnW7IiIkzvYuHdnBujgmsr6cUsWVQ6D8EBfEk6zQaHVaqFTjPP/CLgD0j7XNFb76CDG9HaGnUU7DlNYVlnn9i3fcxTDcC6J/8UA5zYO6/Yfq3GZu4iINyjsiDRxvxzWgYeGd2LylcdHNd11Owt3OmtleiaEYbGcPHV1IvdKrt2HnXU7+1zTWO2jg+keH0rHmGAq7A5mn8GKqiW7nG0Y1jmGdtFBJEYFUmk3WLHnaJ3fQ0TkXFLYEWniWoX489So7sSGHh+56egKOxVVztqb6vU6p9LFHXayi6iocnDgmHNkp0OrYCwWC+MvaAvAB4v31mlFlWEYZr3OxV1bYbFYuKRLDACLdx0+1UtFRBqNwo5IM+Qe2XGr68qnzq4i5V05RRw4VoLDgCA/GzGhzl2d70hpR2iAD7tziuq0OeC+oyUczCvFz2ZlcIcoAC7p6go7OxV2RKRpUNgRaYbcdTxudR3ZcU9j7TlcZK7EahcdbE6BhQX4cs9F7QF4Z8FuDOPUoztLXaM3A9pFEuTnXK5+UadofKwW9h0tId1VEyQi4k0KOyLNUKCfjdauguQAXysdY+q2ArFddBC+NgslFXaWu2pqOrQK8rjnnqEdCPKzseVQAQt3nHp0ZpFrifmwLq3M50IDfLkgyVlMvWS3RndExPsUdkSaKfeKrO7xYdhq2FenJr42qzkqNGercz+ddieMEkUG+3GHa+flt+fvqnV0p7TCbm4meFk3zwN5L+ocDaAiZRFpEhR2RJqpjjHOkFLXKSw3907K6bmu4uQTwg7AvRd3wN/Hyob0PGZtqnnfnaW7j1BW6aBNRCA9Wod6XEvp6Aw7K/cePe1UmIjIuaawI9JM3ZXSnnH9EvilayO/unIXKbudWOwMEBsawF2u2p3JX6TyUw3FynO2Op+7MjnupGXv/ZIiCPC1cqSogl3VjqcQEfEGhR2RZqpLXChv3dLf3CiwrtxFym7to4NqvO/Jkd24pk9rKu0GE6et9wg8dofBPNcp51cmx530Wn8fGwPbOVdnLd+toyNExLsUdkTOM12qhZ3gasvOT+Rjs/Lmzf0Y2zeBSrvB/32+nuV7nMFlQ/oxjhZXEBbgw4WuJecnSunkqtvZq7odEfEuhR2R80yHVsG465mrLzuviY/Nyhs39eXqPq2pchg8/t+fKSirZI7rsNDLusfia6v5x4g77Kzcm1unDQpr8t3Ph3h6+kZKKnT0hIjUn8KOyHkmwNdGUpRz6qp9q5qnsKrzsVl5ZXwfkqKCOJRfxgvfbTVXctU0heXWp004If4+5JdWmiezn4mcgjKe+Opn/r06g3+t3H/GrxcRcVPYETkPuYuUT9ycsDbB/j785aa+WCzw1boD7D1cjK/NwqWu3ZJr4mOzMsh1eOnKekxlTZ2/m7JK53EYny7fT5XdcdI9hmHwwndbGf/ecgrO4PBSETm/KOyInIduvTCRXm3CuKZPQp1fM6h9FA9c3NF8PKRjNKEBvqd8zUWdnJsNLj/D/XbSj5bw79XpgHPTxIN5peZoUnXvLtjNP5alsW7/MebWcF1EBBR2RM5LV/SIY+avLyb5DPfomXxlV7rFOUeFxvRufdr73XU7q9NyaxyZqc0bc3dS5TC4pGsM9w1zBqyPl+3zuGf2pkxe+2mn+XjZ7qZdCP32vF28M3+Xt5shcl5S2BGROgvwtfGv+wbzxs19uWlg4mnv79E6jLAAH4rKq+p8Cvr2rAK+ST0IwBNXdeOOlHb4WC2s3pfLpgP5AGw8kMej//0ZcJ7LBbBiz5Emu4Hh0aJyXp+zk9d+2qnpNhEvUNgRkTMSE+rP9f3b1umICpvVwqhe8QD83+frWbA955T3b8ss4JH/pGIYMKZ3PL3bhhMXFsA1fZyjSG/M3cmj/03l+r8up7TSziVdY/j0lxfiZ7NyKL+MfU304NGMY6Xm9zkF5V5sicj5SWFHRM6p56/txfBuMZRVOrj/s7V86xq1qa60ws7Ls7dzzdSlbM8qJCzAhydGdjev3zPUuUv0/O05TF9/ELvD4LJuMUy9tT8h/j70T4oAYFkT3cAwI/d4CMspLPNiS0TOTwo7InJOBfrZ+PDOgVzXL4Eqh8Fv/pPK3R+vZtXeoxSUVfLXhbu5+JUFvL9oD3aHwehe8cx59FLzoFOAvokRDO/mXPl1VXIc304cysf3XEh4oLNAemhndyF00ww7B6qN7Bwu1MiOSGPz8XYDRKTl87VZef2mfsSGBfD3JXtZuOMwC3ccxs9mpcJVuNwmIpA/jk3mqp7xNb7H+7cPoLi8iuiQk3d8Hto5mtfnOE9ZdzgMrHU8Bb6xZByrNrKjaSyRRqewIyKNwmq18LsxPZgwOIkPFu/ly3UHqKhy0Dk2hIcu7cS1/RJq3Y0ZnMXRAb62Gq/1aRtBsJ+NYyXODQx7tQk/V92oF01jiXiXwo6INKp20cH86freTL6yK1n5ZSS3DjvrkRhfm5ULO0SxYMdhlu850uTCzsHqBcqaxhJpdKrZERGvaBXiT6824Q025eSu22lq++04HIZHzY6msUQan8KOiLQI7t2aV6flUlFV9w0Mz7XDReVmXRJoGkvEGxR2RKRF6B4fSqsQP0or7U1qp2J3vY57AEvTWCKNT2FHRFoEq9XCk6Oce/O8PX83X67NMK85HAaVZ3BcRV5JBYfySk9/Yx24p7C6xzuP5igsq6Ks0t4g7y0idaMCZRFpMW4amMi+I8X8deEenp6+iSqHwY6sQmZtyqTS7uD92wcwpGP0Kd/jpy1ZPPblz9gdBoufvIxWNSx1PxPukZ2eCWHsPlxERZWDw4XlJEYFndX7AizddYSoYL8zPuNM5HyjkR0RaVEev6ob1/RpTZXD4Onpm/hk+T4OF5aTV1LJnf9YzQ+bs2p8XZXdwcuzt/PAP9dRWFZFSYWdhTvqdp7Xqbj32EmMCiI21BmcGqJuJyO3hDv+sYpffrLmrN9LpKVT2BGRFsVqtfDaL/pycZdWBPvZuK5fAh/eOZArk+OoqHLwf5+v49+r0z1eYxgGE6et5/1FewDM3ZsX7zz7sOOexmobGXg87DTAiqzUjDwMA7IKyiipqDrr9xNpyTSNJSItToCvjc9+eSEAFouzMviybjE8881m/rMmg6enb6J7fCj9k5wnpq/Zd4wft2TjZ7Py+s19iQsL4Bfvr2DJrsPYHUadDj2tjefITgDQMEXKWw4VmN9n5ZfRMSbkrN9TpKXSyI6ItEgWi8UMOgA+NitTbujNtX0TAHh9zk7zmntEZ/yAtlzTJ4H+iRGEBvhwrKSSTQfz692GKruDzDznlFXbyEBiw06expq9KfO0p8HXZMuh4+3KytdydpFTUdgRkfOGxWLhiZHd8LFaWLLrCKv2HmVbZgHzt+dgtcCvLukIOIPRMNcmhYvOom4nq6CMKoeBn81KXGjASdNY2QVlTJy2nvs/W8uRorqP9hiGwdZqIzuZCjsip6SwIyLnlcSoIG4elAjAX37ayd9cozqje7emfbWT1i/t6jxlfdHOmkddSiqqKC4/da1MRq6zXqdNZCBWq+WkaazUjDwcBlQ5DOZuza5zH7IKyjhaXOHxWERqp7AjIuedSZd3xs/Hyup9uXyTegiAhy7t5HHPJa6wk5qRR35Jpce1o0XljHxzMZe9tpCiUwSeA656nbaRgQDEmNNYzrCz8UCeee8PW2peJVaTLQcLPB5rGkvk1BR2ROS80zo8kAmDk8zHF3dpddLhoQkRgXSNC8FhwNLdR8znHQ6Dx7/8mYzcUnIKy5l/inqbDHMllnNPHfc01mFXzc7GA8frbpbtPkJ+aSV14S5O9nEVTmsaS+TUFHZE5Lz00PBOBPranN+fMKrjdkmXk6eyPlqaxoJqdTw/nrBvzyfL0nh73i7KKu0njey4p7GOFldQaXeYYSfYz0al3ahzobK7OPnCDlGAs/ZHRGqnsCMi56XY0AA+u/dC3rmtPxe5ipFPdGk3Z9iZty2Hf69O59vUg/z5h+0A3Hqhc2RowY4c8/iHTQfyee67rbw+Zye3fLDSLCJ275YcFeyHzWrBMGBDeh75pZX42azcntIOoNYND0/kHtm5okccoJEdkdNR2BGR89ag9lFc0yfhlNdD/H04WlzB09M38Zv/pFLlMBjTO56Xru9Fm4hASirs5uaDHy9LM1+bmpHH9qxC4PjIjs1qITrYD4B525wFyT1ahzLW1YaFO3NOu0HgseIKDrrO7bqieywAR4vLm9RJ7yJNjcKOiEgtAnxtfHzPIO4b1oFLu8bQJiKQfokRTLmhDxaLhZE94wHniExOQRnfbXQWO791Sz+6xB7f5C8x8vg5WO69dua4wk6fthH0TAgjMSqQskrHaXdt3prpHNVJigqiXXQQfjYrhlH7ERQ5hWUUltWtFkikpdIOyiIipzCofRSD2kfVeG1Ur3j+sSyNuduyiQ0LoNJuMKBdJOP6teHy7rG89P02AnxtxIQeP0zUWbdTwN7DxQD0aRuOxWJhVM94PlySxg+bsxjVq3Wt7XHX6/RMCMNisRAX7k9GbilZ+WVmIbRbVn4ZV/xlId1bh/H1Qxed5Z+ESPOlkR0RkXoa0C6SViH+FJRV8eGSvQD8cmgHAEIDfJlyQx/+OLanx2tiQz1PUe+bGAFgBpwftmR5LEk/0WbXsnP36rHWYc4pspr22lmVdpTiCjvr9h8zi6VFzkcKOyIi9WSzWriqp7NI2O4waBMRyEjX49pUDztBfjY6uc606p8YwbDOrSirdHDXP1azK7uwxte7R3aSE8IAiA93rvCqaa+dE5e2i5yvFHZERM7C6F7x5vd3XdQOH9upf6zGhAWY3/dKCDcPGbVaLbx/xwD6tg3nWEklt3+0iozc46MxlXYH09cfYO8R5/RXzxPCTk0rsqqf67VkV/3DzuHCcka9uZg3qp0nJtKcqGZHROQsDOkYTfvoIEoq7Nw8MOm091cf2enT1nMjwxB/Hz6550Ju/mAFO7OLGP7aQjrFBNMtPoy1+3LNQNMtLtTcsyfeFZ5OnMZyOAy2VAs7y/ccxeEwsFY7wb3K7mBHdiHbMgvpnxRhjjKdaObGQ2zPKiQjt4RJl3fG9zSBTqSpUdgRETkLvjYrsx6+GIdhEBrge9r7q4ed3ieEHYDIYD/+ee9g7vl4DVszC9iZXcTO7CIAYkL9uWdoeyYMbmfe37qWaay9R4oprrAT4GvFZrGQW1zB1swCerUJp7i8iie/2si87dmUVTrM917w+HBC/E/+tbDUNSpUXGFnQ3qeuZmhSHOhsCMicpaCawgItYmtNo3Vt21EjffEhQUw6+FhZBeUs+VQPtuzCokPC+Cavq3x97F53ltL2NnsGtVJbh1GZJAf87bnsHT3EXq1CefvS9KYtSkTgNAAHyw4p6r+tmgPj13VzeN9Ku0OVu49aj5esuuwwo40OxqLFBFpRPFhAfRPiiClYzTtooNqvc9isRAfHsAVPeKYeFlnxg9oe1LQgeMjO9kFZTgchvm8u16nT9sIhnVx7hC9dNcRjhVXmCvHXrmxDz//4SpeubEvAB8s3mtuWOi2IT2P4gq7+XjxWdT+lFfZeXfBbrZlFpz+ZpEGpLAjItKIbFYL0x+6iH8/MASLxXL6F5xGTIg/VgtUOQyOFJebz7vDTq824VzsCjur9+Xy5tydFJVXkdw6jBsvaIvVamFkzzgGd4iivMrBK67jMNyW7nJucniha6+hTQfyyCupqFdb/7F0H6/+uIPJX6TW6/Ui9aWwIyLSyBoi5Lj52KzmpoXuqazqxcm924TTKSaE+LAAKqocfLpiPwBPjOxmFitbLBaevSYZiwW+TT3EhvRj5vsvcS1ZHz+gDV1inafAL99zfFqrJkXlVbw8ezurqk1/2R0G/1rp/OztWYXmuWEijUFhR0SkmYsPd20s6Ao77uLkQF8bnWKCsVgsDK122OnAdpEMdx1y6tarTTjjL2gLwB//t4VKu4P80kp+zsgDYFiXGC52nQJ/umXsL363lfcX7eGhz9dT4DqqYsH2HI8psunrD5xFj0XOjMKOiEgzF+86b8u9/NwsTk4IM/f9cU9lgXNUp6bRpSdHdiMswIeNB/KZOn83K/YcxWFAx5hg2kQEmu+xZNdhDMM46fXgDDVfrM0AILe4gqnzdgHwT9eoTo/Wzv2Bvkk9RJVdh5dK41DYERFp5lqfMLLj3jm5d5vjS9sv7xFL7zbh3DY4icEdo2t8n9iwAP50fW8A3pm/yyxkvtg1KjS4YxS+NgsHjpWy/+jJx0/kl1Ty2+kbAcwVW58s38fCHTks2nkYiwWm3tqfyCBfjhSVs/Q0uzpvPJDHHR+t4tvUg3X7gxCpRZMOO8899xwWi8Xjq3v37ub1srIyJk6cSHR0NCEhIYwfP57s7GwvtlhEpPGdeGTE5oMnh52wAF+++/UwXnKFmdqM7ZvA9f3b4DBg3X5n7c4w1/RVkJ8PA9pFAvBN6kHmbcvmw8V7+XT5Pr7flMnvZmwiu6Ccjq2C+fSeCxneLYZKu8ED/1wHwKVdY+gcG8K1fRMAmL6+9hCzOi2X2z5cxZJdR3jki9RmH3gcDqPGIz2kcTT5fXZ69uzJ3Llzzcc+PsebPHnyZGbNmsWXX35JeHg4kyZN4oYbbmDZsmXeaKqIiFe4d1E+lF9KRm6JeX5WTZsW1sXz43qyOi2Xg3ml2KwWhnQ8vq/OxV1iWLk3lzfn7qrxtVYLvPqLvgT62Xjm6h4s2XWEiirndNWdKc7NEG+4oC2frtjPj1uyKCyrPGkzxiW7DnP/Z2spq3TQKsSfI0XlPPrfnwn0tXFVz/iTPrM5mDJ7Gx8uSePDOwdyZfKpz0+ThtekR3bAGW7i4+PNr1atnMOp+fn5fPTRR7z++utcfvnlDBgwgI8//pjly5ezcuVKL7daRKTxuEd2Vu7N5eJXFlQrTq75+IfTCQvw5Y2b+xHga2VEj1iPMHJNn9YE+9nws1npHh/K1X1aM7pXPAPbRdIxJpinR/cwR386x4ZyxxBnwGkbGcilXWMB5zEZnWKCKa9yMHtTlsdnb0g/xr2fOIPO8G4xLH5yODdc0Aa7w2DStA0s39P8DjTNKSjj0+XOmqVvNjTvEarmqsmP7OzatYuEhAQCAgJISUlhypQpJCUlsW7dOiorKxkxYoR5b/fu3UlKSmLFihUMGTKk1vcsLy+nvPz4fhQFBVoCKSLNV5fYEAJ9bZRW2vGzWekcG8LtQ9qZh4zWx4Udolj19AiC/T03MmwXHcyGP1yF1cJpDz0FeHxkN/x9rIxIjjPbY7FYuOGCtrz64w4+WprG9Re0wddmxeEweO5/W6iwO7iieyx/vf0C/H1svDK+D6UVdmZvzuKZGZv5afIldfrspuIfy/ZR4SrGXrr7CHaHcVZ/N3LmmnTYGTx4MJ988gndunUjMzOT559/nosvvpjNmzeTlZWFn58fERERHq+Ji4sjKyur5jd0mTJlCs8///w5bLmISOOJDvFn3mOXUlBWSaeYkAY7qDM8qOazvvx86v7+If4+PD2mx0nP33ZhEn9fspcd2YV8tDSNBy/txHcbD/HzgXyC/Wy8PL6PuWO0j83Kq7/oy8q9R9l7pJhvUw8xfkDb+nWqkRWUVfK5ayWaxYJzOf+BPC5IivRyy84vTToajx49ml/84hf06dOHkSNH8v3335OXl8d///vfs3rfp59+mvz8fPMrIyOjgVosIuIdCRGBdI8PazYnkkcG+/H7q5MBeHPuTnbnFPLn2c7dm//vss7mRoluIf4+/OrSTgC8NW8XlQ2wbH11Wi6XvLKARTsPn9X7bD6Yz2P//ZmM3JNXqP1r5X4Ky6voGhfCyGRnvdHis/w8OXPN41+FS0REBF27dmX37t3Ex8dTUVFBXl6exz3Z2dnEx5+6gM3f35+wsDCPLxERaVzjL2jDkI5RlFU6GP/eCg7ll5EQHsC9wzrUeP+dKe1oFeJHem6JuSlhZn4pb87dyZNf/cwvP1nDzX9bcVJdz4o9Rxn68nx+3OI56v/Zin2k55bw/sI9dWpvUXkV32w4SElFlflcaYWdhz5fx9frD/DkVxs99h8qq7Tzj6X7AHjw0k7mRo4KO42vWYWdoqIi9uzZQ+vWrRkwYAC+vr7MmzfPvL5jxw7S09NJSUnxYitFRKQuLBYL/++63vjaLOSXOndafmJUNwJ8Tz7wFJxL3x90je68PW83b87dyeWvLeLNubv479oDzN+ew6q0XP7w7RbzUFTDMHh59jYO5pXy8bI0870Mw2Dl3lzAeWZY9fO+isur+HJtBsXlVVT31NcbeeSLVO7/bK25wuyNuTvJyHXuDL1i71F+2Hw8UH2+Kp0jReW0iQhkbN8ELunqDDupGXnkl1TW/w9OzliTDjuPP/44ixYtYt++fSxfvpzrr78em83GrbfeSnh4OPfeey+PPvooCxYsYN26ddxzzz2kpKScsjhZRESajs6xITw0vDPgXKU1rm+bU95/+5B2xIb6czCvlDfn7qK00s7AdpE8MbIbU27oTWiAD7tzipi3PQdwTlX97Npkcd3+Y+aozJ7DxRwpci5UsTsM5rvuB/jT99t44itnsHGP1KRm5DFrYyYAy3Yf5bfTN7LpQD5/d228OMy18eL/m7WN0go787dn89L32wD41aUd8bVZSYgIpLPrfLFlzXBVWXPWpMPOgQMHuPXWW+nWrRs33XQT0dHRrFy5kpgYZzp+4403uOaaaxg/fjyXXHIJ8fHxTJ8+3cutFhGRM/GbK7ow9db+/P2ugebhpLUJ8LXx6JVdAWgTEci7t13Alw+mMPGyztx6YRK3u5a6v7/IOTX14ZLjozmVdoNVrtGclXs9DzP9aYtzQ9r8kkpzimzO1mz+9/MhDMNgiiu49E+KcJ5cv/4gt324EofhXI7/4Z0DSQgP4GBeKY99mcpD/1qP3WFwff823D64nfk5l3TRVJY3NOnVWP/5z39OeT0gIIB3332Xd999t5FaJCIiDc1mtTDWtatyXdxyYRIXdogiISLwpCmvey5qz0dL0li3/xhfrEln7jZniBnSMYqVe3NZsusIl3WPZVWaM/RcmRzHnK3ZLNp5mLJKO1+uy6Cs0oGvzUKl3eCP/9tCWaWdVWm5+PtYefe2C1iy6zBPfb2JwvIqwgN9+ePYngT62fj91clMnLae7117B43oEcsrN/bxCHCXdG3FP5alsXin83yxms4oa65e/XE7O7IKeee2C2qdivSWJj2yIyIiUpOOMSE1/kKNDQtg/ADnVNjT0zcBztBxx5D2wPFDTN0jO/cO60BCeACllXYW7zzMZyucy8T/cE0yya3DyCup5Kmvne9zz9AOJEQEcvOgJJ4c1Y3IIF+m3NDbXDk2pne8udv0he2jeOe2C05aHTekYzT+PlYO5Zex53BRA/+peM/BvFL+unAPc7flNMmNHxV2RESkRbn/4o5YLOCqUeb+izsytHM0FgvsyilixZ6jHC4sx8/HSr/ECPMIipe+30Z6bglhAT6MH9CWV3/RBx/XqExEkC8PDe9kfsb/De/MumeuZEzv1uZzFouFD+4cyBs39+XjewbVGMYCfG3mIamLdza9UFBfM9YfwL0QzT1V2JQo7IiISIvSMSaEUa4A06dtOBd2iCIiyI8+roNRX/tpBwAXJEUQ4GvjKtdZVftcJ7nfNDCRID8feiaE89hV3QB4cmR3wgM9N1msqb4oLMCX6/u3Jdi/9iqRoa5i5hUn1A3VZsWeozz51c+kHSmu9Z69h4v4bMU+Cssaf5WXYRh8Xe1Q15VpTS/sNOmaHRERkfr4/dU98Pexct/FHc26mIu7xPDzgXzWp+cBziklgEEdoggP9CW/tBKLBbPIGeCh4Z2YMCSJsICad5OuD/fnrtp7tE5HR/zp+61sPljA7E1ZvHZTX0aecBhqWaWdO/+xmgPHSnlv4R5euqE3l3WLPel9Xp+zk9mbMvns3gtpHR7YYP1Zn55H2pFi/GxWKuwONh/Mp6i8ipBTBL7GppEdERFpcdpGBvHmLf3p1eb4ye/DurTyuGdwB2fo8LVZuby7Mxxc2jWG9q2CPe5ryKAD0CshjBB/HwrKqtiWeeqzGQ8XlrP5oPOewvIqfvXPdUz5fht2x/HNCz9bsY8Dx5x7/WTml3HPx2t4/Mufzb2AAHKLK3h/4R525RQxbVW6x2dsyyzgy7UZHhsinomvXavXrunTmraRgdgdBuv2H6vXe50rCjsiInJeuCApkiA/Zx2Nn4+V/kkR5rVHr+zKjQPa8sexPc95O3xsVrNu58Ql8Cdassu5RL1H6zDuc+0s/bfFe3nmm00YhkFeSQXvzN8NwAvjenLvsA5YLPDVugN8tmKf+T7T1x8wDyOdseGgGWwqqhzc8/EanvhqIwvrsRy+rNLOzJ8PATB+QFszQK6q4xRdY1HYERGR84Kfj9WcQuqfGOFRQJwYFcRrv+hLhxNGdc4V96qtFXtOHQrc53Zd3j2GZ65J5q1b+mG1wL9XZ/D6nJ1Mnb+bgrIquseHMmFwO569JpkXx/UCnKGorNKOYRhMW318NOfAsVLWukZevt+USVZBGQDzt+VwpuZuy6agrIqE8ABSOkYzuGPdQlxjU9gREZHzxs2DEgG40cunpqd0dE6prU7L9ZiSqs7uMMzNBy/t6pxmG9evDf/vut4ATJ2/2zwC43djepi1PzcNTKRNRCCHC8v5Yk0Gq9Ny2Xu4mCA/m1m47R7d+Ue1IzQW7Mg546msr9c5p7Cuv6ANVquFIa6RnY0H8j3OEPM2hR0RETlvjOwZz64/jeYXAxO92o7khDBCA3woLK9iy6F88/nqYWPzwXyOlVQS6u/jMeV22+AkcxdphwEXd2llnrsFzhGsB13L5N9ftMfcO+javgnckeIsvp61MZPle46y8UA+/j5W/GxWDhwrZc9hzxVfjlqCGEDakWJz6mv8Bc7wmBgVSEJ4AFUOg/X78870j+WcUdgREZHzyokb/XmDzWphcIfjU1nuIykG/WmuOZrj/t+hnVud1OZfX96Zh4Z3IikqiD9ck3zS+/9iQFviwvzJzC9j1ibnmV63XpjEkI7RxIcFkF9ayeQvUgG4vn8bc/pp4Y7jU1mP/GcDnX7/Pf1f+IkRry/iya9+9jgc9aOlezEMuLx7LB1jQgDnXkOD3avN0prOVJb3/8ZFRETOQ+76oRV7j/Leoj38bfFejhRV8PB/NnAor9Ss17m0W8xJr7VYLDw1qjuLn7yMLnGhJ10P8LXxq0uOb4KY3DqMPm3DsVktjOvvPJojp9B5EOovh3VguGup+sIdzs9cuy+Xb1IPYRhwrKSS3TlF/HftAV75YTvgXN31lWsK6/6LO3p8tjvENaXNBRV2REREvCClkzPsLNt9hFd+cG50GBPqT15JJQ/+ax0bMvIAPKaozsStFybRKsTP9X2iud/QDf2P1ytd3KUVXeNCGe4KVKvTcikur+KNuTsB5/TUT5Mv4ZXxfQD4dMV+Vqfl8q+V+ymrdNCrTZhZbO3mHtlJzcijwAubHNZEYUdERMQLesSHER7oS6XdWRdz37AOfP3gRYQG+LDxQD52h0GX2BDaRNRvA8BAPxvv3T6AR0Z04eZBSebz3eJD6ZcYARwflenYKpikqCAq7A7emreLZbuP4muzMPnKLnSNC+WmQYnc7KpzevKrn81l7fdX27TRrX10EO2ine/1wGdrKau016v9DUlhR0RExAusVgsXuzY6HNM7nt+N6UFSdBCv3tjXvOfSeo7quA1qH8UjI7ri5+P56/7vdw3km4lDzVEji8Viju58sHgvAL8YmEjbyCDzNb+/pgfxYQHsO1rCkaIKEsIDPM4Gc7NYLEy9tT8h/j6s3JvLQ/9a57HBoTco7IiIiHjJH8f25O1b+/PGzf3Ms7ZG9YrnN1d0oVWIHzcOPDdL5FuF+JujO27Vj5jws1mZeFlnj+thAb68dEMv8/Evh3Wotdi7T9sI/nH3IAJ8rSzYcZjJX6TWusS+MSjsiIiIeElMqD/X9k3A38fzhPTJV3Zl7TNX0j0+rNHaMqRjtDkCdPOgxBqnzy7vHscjI7pwZXIct16YdNL16i7sEMXf7hiIn83KrE2Z/LQl65y0uy6azildIiIi4jWBfjbuG9aBxbsO8+vLO9d63yMjutb5PS/tGsPbt/Znz+EiRtcw5dVYLEZ9T/5qQQoKCggPDyc/P5+wsMZL0SIiIlJ/df39rWksERERadEUdkRERKRFU9gRERGRFk1hR0RERFo0hR0RERFp0RR2REREpEVT2BEREZEWTWFHREREWjSFHREREWnRFHZERESkRVPYERERkRZNYUdERERaNIUdERERadEUdkRERKRF8/F2A5oCwzAA51HxIiIi0jy4f2+7f4/XRmEHKCwsBCAxMdHLLREREZEzVVhYSHh4eK3XLcbp4tB5wOFwcOjQIUJDQ7FYLN5uToMqKCggMTGRjIwMwsLCvN2cc+586u/51FdQf1u686m/51Nf4dz21zAMCgsLSUhIwGqtvTJHIzuA1Wqlbdu23m7GORUWFnZe/KNyO5/6ez71FdTflu586u/51Fc4d/091YiOmwqURUREpEVT2BEREZEWTWGnhfP39+ePf/wj/v7+3m5Kozif+ns+9RXU35bufOrv+dRXaBr9VYGyiIiItGga2REREZEWTWFHREREWjSFHREREWnRFHZERESkRVPYaQGmTJnCoEGDCA0NJTY2luuuu44dO3Z43FNWVsbEiROJjo4mJCSE8ePHk52d7aUWN6yXX34Zi8XCI488Yj7X0vp78OBBbr/9dqKjowkMDKR3796sXbvWvG4YBn/4wx9o3bo1gYGBjBgxgl27dnmxxfVjt9t59tln6dChA4GBgXTq1IkXX3zR49yb5tzXxYsXM3bsWBISErBYLHzzzTce1+vSt9zcXCZMmEBYWBgRERHce++9FBUVNWIv6u5U/a2srOSpp56id+/eBAcHk5CQwJ133smhQ4c83qOl9PdEDz74IBaLhTfffNPj+ebS37r0ddu2bVx77bWEh4cTHBzMoEGDSE9PN6835s9phZ0WYNGiRUycOJGVK1cyZ84cKisrueqqqyguLjbvmTx5Mt999x1ffvklixYt4tChQ9xwww1ebHXDWLNmDX/729/o06ePx/Mtqb/Hjh1j6NCh+Pr6Mnv2bLZu3cpf/vIXIiMjzXteeeUV3n77bd5//31WrVpFcHAwI0eOpKyszIstP3N//vOfee+993jnnXfYtm0bf/7zn3nllVeYOnWqeU9z7mtxcTF9+/bl3XffrfF6Xfo2YcIEtmzZwpw5c5g5cyaLFy/mgQceaKwunJFT9bekpIT169fz7LPPsn79eqZPn86OHTu49tprPe5rKf2tbsaMGaxcuZKEhISTrjWX/p6ur3v27GHYsGF0796dhQsXsnHjRp599lkCAgLMexr157QhLU5OTo4BGIsWLTIMwzDy8vIMX19f48svvzTv2bZtmwEYK1as8FYzz1phYaHRpUsXY86cOcall15q/OY3vzEMo+X196mnnjKGDRtW63WHw2HEx8cbr776qvlcXl6e4e/vb/z73/9ujCY2mKuvvtr45S9/6fHcDTfcYEyYMMEwjJbVV8CYMWOG+bgufdu6dasBGGvWrDHvmT17tmGxWIyDBw82Wtvr48T+1mT16tUGYOzfv98wjJbZ3wMHDhht2rQxNm/ebLRr18544403zGvNtb819fXmm282br/99lpf09g/pzWy0wLl5+cDEBUVBcC6deuorKxkxIgR5j3du3cnKSmJFStWeKWNDWHixIlcffXVHv2Cltff//3vfwwcOJBf/OIXxMbG0r9/fz788EPzelpaGllZWR79DQ8PZ/Dgwc2uvxdddBHz5s1j586dAPz8888sXbqU0aNHAy2rryeqS99WrFhBREQEAwcONO8ZMWIEVquVVatWNXqbG1p+fj4Wi4WIiAig5fXX4XBwxx138MQTT9CzZ8+TrreU/jocDmbNmkXXrl0ZOXIksbGxDB482GOqq7F/TivstDAOh4NHHnmEoUOH0qtXLwCysrLw8/Mzf4C4xcXFkZWV5YVWnr3//Oc/rF+/nilTppx0raX1d+/evbz33nt06dKFH3/8kYceeoiHH36YTz/9FMDsU1xcnMfrmmN/f/vb33LLLbfQvXt3fH196d+/P4888ggTJkwAWlZfT1SXvmVlZREbG+tx3cfHh6ioqGbf/7KyMp566iluvfVW87DIltbfP//5z/j4+PDwww/XeL2l9DcnJ4eioiJefvllRo0axU8//cT111/PDTfcwKJFi4DG/zmtU89bmIkTJ7J582aWLl3q7aacMxkZGfzmN79hzpw5HvO/LZXD4WDgwIG89NJLAPTv35/Nmzfz/vvvc9ddd3m5dQ3rv//9L59//jnTpk2jZ8+epKam8sgjj5CQkNDi+irHVVZWctNNN2EYBu+99563m3NOrFu3jrfeeov169djsVi83ZxzyuFwADBu3DgmT54MQL9+/Vi+fDnvv/8+l156aaO3SSM7LcikSZOYOXMmCxYsoG3btubz8fHxVFRUkJeX53F/dnY28fHxjdzKs7du3TpycnK44IIL8PHxwcfHh0WLFvH222/j4+NDXFxci+pv69atSU5O9niuR48e5qoGd59OXMXQHPv7xBNPmKM7vXv35o477mDy5MnmCF5L6uuJ6tK3+Ph4cnJyPK5XVVWRm5vbbPvvDjr79+9nzpw55qgOtKz+LlmyhJycHJKSksyfW/v37+exxx6jffv2QMvpb6tWrfDx8Tntz63G/DmtsNMCGIbBpEmTmDFjBvPnz6dDhw4e1wcMGICvry/z5s0zn9uxYwfp6emkpKQ0dnPP2hVXXMGmTZtITU01vwYOHMiECRPM71tSf4cOHXrSVgI7d+6kXbt2AHTo0IH4+HiP/hYUFLBq1apm19+SkhKsVs8fSzabzfx/ii2pryeqS99SUlLIy8tj3bp15j3z58/H4XAwePDgRm/z2XIHnV27djF37lyio6M9rrek/t5xxx1s3LjR4+dWQkICTzzxBD/++CPQcvrr5+fHoEGDTvlzq9F/LzV4ybM0uoceesgIDw83Fi5caGRmZppfJSUl5j0PPvigkZSUZMyfP99Yu3atkZKSYqSkpHix1Q2r+mosw2hZ/V29erXh4+Nj/OlPfzJ27dplfP7550ZQUJDxr3/9y7zn5ZdfNiIiIoxvv/3W2LhxozFu3DijQ4cORmlpqRdbfubuuusuo02bNsbMmTONtLQ0Y/r06UarVq2MJ5980rynOfe1sLDQ2LBhg7FhwwYDMF5//XVjw4YN5uqjuvRt1KhRRv/+/Y1Vq1YZS5cuNbp06WLceuut3urSKZ2qvxUVFca1115rtG3b1khNTfX42VVeXm6+R0vpb01OXI1lGM2nv6fr6/Tp0w1fX1/jgw8+MHbt2mVMnTrVsNlsxpIlS8z3aMyf0wo7LQBQ49fHH39s3lNaWmr83//9nxEZGWkEBQUZ119/vZGZmem9RjewE8NOS+vvd999Z/Tq1cvw9/c3unfvbnzwwQce1x0Oh/Hss88acXFxhr+/v3HFFVcYO3bs8FJr66+goMD4zW9+YyQlJRkBAQFGx44djd///vcev/yac18XLFhQ47/Vu+66yzCMuvXt6NGjxq233mqEhIQYYWFhxj333GMUFhZ6oTend6r+pqWl1fqza8GCBeZ7tJT+1qSmsNNc+luXvn700UdG586djYCAAKNv377GN9984/Eejflz2mIY1bYmFREREWlhVLMjIiIiLZrCjoiIiLRoCjsiIiLSoinsiIiISIumsCMiIiItmsKOiIiItGgKOyIiItKiKeyIiIhIi6awIyJe0b59e9588806379w4UIsFstJBweKiJyOdlAWkToZPnw4/fr1O6OAciqHDx8mODiYoKCgOt1fUVFBbm4ucXFxWCyWBmnDmVq4cCGXXXYZx44dIyIiwittEJEz5+PtBohIy2EYBna7HR+f0/9oiYmJOaP39vPzIz4+vr5NE5HzmKaxROS07r77bhYtWsRbb72FxWLBYrGwb98+c2pp9uzZDBgwAH9/f5YuXcqePXsYN24ccXFxhISEMGjQIObOnevxnidOY1ksFv7+979z/fXXExQURJcuXfjf//5nXj9xGuuTTz4hIiKCH3/8kR49ehASEsKoUaPIzMw0X1NVVcXDDz9MREQE0dHRPPXUU9x1111cd911tfZ1//79jB07lsjISIKDg+nZsyfff/89+/bt47LLLgMgMjISi8XC3XffDYDD4WDKlCl06NCBwMBA+vbty1dffXVS22fNmkWfPn0ICAhgyJAhbN68+bSfKyJnT2FHRE7rrbfeIiUlhfvvv5/MzEwyMzNJTEw0r//2t7/l5ZdfZtu2bfTp04eioiLGjBnDvHnz2LBhA6NGjWLs2LGkp6ef8nOef/55brrpJjZu3MiYMWOYMGECubm5td5fUlLCa6+9xj//+U8WL15Meno6jz/+uHn9z3/+M59//jkff/wxy5Yto6CggG+++eaUbZg4cSLl5eUsXryYTZs28ec//5mQkBASExP5+uuvAdixYweZmZm89dZbAEyZMoXPPvuM999/ny1btjB58mRuv/12Fi1a5PHeTzzxBH/5y19Ys2YNMTExjB07lsrKylN+rog0gHNylrqItDiXXnqp8Zvf/MbjuQULFhiA8c0335z29T179jSmTp1qPm7Xrp3xxhtvmI8B45lnnjEfFxUVGYAxe/Zsj886duyYYRiG8fHHHxuAsXv3bvM17777rhEXF2c+jouLM1599VXzcVVVlZGUlGSMGzeu1nb27t3beO6552q8dmIbDMMwysrKjKCgIGP58uUe9957773Grbfe6vG6//znP+b1o0ePGoGBgcYXX3xx2s8VkbOjmh0ROWsDBw70eFxUVMRzzz3HrFmzyMzMpKqqitLS0tOO7PTp08f8Pjg4mLCwMHJycmq9PygoiE6dOpmPW7dubd6fn59PdnY2F154oXndZrMxYMAAHA5Hre/58MMP89BDD/HTTz8xYsQIxo8f79GuE+3evZuSkhKuvPJKj+crKiro37+/x3MpKSnm91FRUXTr1o1t27bV63NFpO40jSUiZy04ONjj8eOPP86MGTN46aWXWLJkCampqfTu3ZuKiopTvo+vr6/HY4vFcspgUtP9xlkuML3vvvvYu3cvd9xxB5s2bWLgwIFMnTq11vuLiooAmDVrFqmpqebX1q1bPep2GvpzRaTuFHZEpE78/Pyw2+11unfZsmXcfffdXH/99fTu3Zv4+Hj27dt3bht4gvDwcOLi4lizZo35nN1uZ/369ad9bWJiIg8++CDTp0/nscce48MPPwScfwbu93FLTk7G39+f9PR0Onfu7PFVva4JYOXKleb3x44dY+fOnfTo0eO0nysiZ0fTWCJSJ+3bt2fVqlXs27ePkJAQoqKiar23S5cuTJ8+nbFjx2KxWHj22WdPOUJzrvz6179mypQpdO7cme7duzN16lSOHTt2yn16HnnkEUaPHk3Xrl05duwYCxYsMANJu3btsFgszJw5kzFjxhAYGEhoaCiPP/44kydPxuFwMGzYMPLz81m2bBlhYWHcdddd5nu/8MILREdHExcXx+9//3tatWplrgw71eeKyNnRyI6I1Mnjjz+OzWYjOTmZmJiYU9bfvP7660RGRnLRRRcxduxYRo4cyQUXXNCIrXV66qmnuPXWW7nzzjtJSUkhJCSEkSNHEhAQUOtr7HY7EydOpEePHowaNYquXbvy17/+FYA2bdrw/PPP89vf/pa4uDgmTZoEwIsvvsizzz7LlClTzNfNmjWLDh06eLz3yy+/zG9+8xsGDBhAVlYW3333ncdoUW2fKyJnRzsoi8h5w+Fw0KNHD2666SZefPHFRvtc7bws4l2axhKRFmv//v389NNPXHrppZSXl/POO++QlpbGbbfd5u2miUgj0jSWiLRYVquVTz75hEGDBjF06FA2bdrE3LlzVQsjcp7RNJaIiIi0aBrZERERkRZNYUdERERaNIUdERERadEUdkRERKRFU9gRERGRFk1hR0RERFo0hR0RERFp0RR2REREpEX7/xWJAPrbcMLdAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.54443359375, recall = 0.3987839771101574, f1 = 0.4603633360858795\n",
      "precision = 0.4899211218229623, recall = 0.3423147581139008, f1 = 0.40302811824080753\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
